{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python       \n",
    "# coding: utf-8\n",
    "\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "\n",
    "from os import path\n",
    "import sklearn\n",
    "from sklearn import metrics, preprocessing\n",
    "from sklearn.metrics import (precision_score, recall_score, f1_score, roc_auc_score, \n",
    "                             roc_curve, auc, confusion_matrix, accuracy_score, \n",
    "                             balanced_accuracy_score, matthews_corrcoef, classification_report)\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder, LabelEncoder, \n",
    "                                   MinMaxScaler, OneHotEncoder, Normalizer, \n",
    "                                   MaxAbsScaler, RobustScaler, PowerTransformer, LabelBinarizer)\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, BaggingClassifier\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.datasets import load_iris, make_classification\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "from collections import Counter\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from lightgbm import LGBMClassifier\n",
    "import catboost\n",
    "import xgboost as xgb\n",
    "from scipy.stats import mode\n",
    "from tabulate import tabulate\n",
    "import shap\n",
    "import joblib\n",
    "\n",
    "start_program = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters to use or load models, but yet to be implemented, it is not really functional."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "load_model_ada = 0 \n",
    "load_model_dnn = 0 \n",
    "load_model_mlp = 0 \n",
    "load_model_lgbm = 0 \n",
    "load_model_rf = 0 \n",
    "load_model_svm = 0\n",
    "load_model_knn = 0 \n",
    "#----------------------------------------------\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pick name of the file according to parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#----------------------------------------------\n",
    "# feature_selection_bit = 1\n",
    "feature_selection_bit = 0\n",
    "\n",
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "    output_file_name = \"ensemble_base_models_all_features.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---- ensemble_base_models_all_features', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    output_file_name = \"ensemble_base_models_feature_selection.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('----ensemble_base_models_feature_selection--', file = f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def confusion_metrics (name_model,predictions,true_labels,time_taken):\n",
    "\n",
    "    name = name_model\n",
    "    pred_label = predictions\n",
    "    y_test_01 = true_labels \n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(name, file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "    FPR = FP / (FP + TN)\n",
    "    FPR = 100*(sum(FPR)/len(FPR))\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Time Taken: ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('FPR: ', FPR, '%' ,file = f)\n",
    "\n",
    "    return Acc, Precision, Recall, F1, BACC, MCC, FPR\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load NSLKDD "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# attach the column names to the dataset\n",
    "feature=[\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\"dst_bytes\",\"land\",\"wrong_fragment\",\"urgent\",\"hot\",\n",
    "          \"num_failed_logins\",\"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\"num_file_creations\",\"num_shells\",\n",
    "          \"num_access_files\",\"num_outbound_cmds\",\"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\"srv_serror_rate\",\n",
    "          \"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\", \n",
    "          \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\n",
    "          \"dst_host_srv_serror_rate\",\"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"label\",\"difficulty\"]\n",
    "# KDDTrain+_2.csv & KDDTest+_2.csv are the datafiles without the last column about the difficulty score\n",
    "# these have already been removed.\n",
    "\n",
    "train='KDDTrain+.txt'\n",
    "test='KDDTest+.txt'\n",
    "\n",
    "df=pd.read_csv(train,names=feature)\n",
    "df_test=pd.read_csv(test,names=feature)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimensions of the Training set: (125973, 43)\n",
      "Dimensions of the Test set: (22544, 43)\n",
      "Label distribution Training set:\n",
      "normal             67343\n",
      "neptune            41214\n",
      "satan               3633\n",
      "ipsweep             3599\n",
      "portsweep           2931\n",
      "smurf               2646\n",
      "nmap                1493\n",
      "back                 956\n",
      "teardrop             892\n",
      "warezclient          890\n",
      "pod                  201\n",
      "guess_passwd          53\n",
      "buffer_overflow       30\n",
      "warezmaster           20\n",
      "land                  18\n",
      "imap                  11\n",
      "rootkit               10\n",
      "loadmodule             9\n",
      "ftp_write              8\n",
      "multihop               7\n",
      "phf                    4\n",
      "perl                   3\n",
      "spy                    2\n",
      "Name: label, dtype: int64\n",
      "\n",
      "Label distribution Test set:\n",
      "normal             9711\n",
      "neptune            4657\n",
      "guess_passwd       1231\n",
      "mscan               996\n",
      "warezmaster         944\n",
      "apache2             737\n",
      "satan               735\n",
      "processtable        685\n",
      "smurf               665\n",
      "back                359\n",
      "snmpguess           331\n",
      "saint               319\n",
      "mailbomb            293\n",
      "snmpgetattack       178\n",
      "portsweep           157\n",
      "ipsweep             141\n",
      "httptunnel          133\n",
      "nmap                 73\n",
      "pod                  41\n",
      "buffer_overflow      20\n",
      "multihop             18\n",
      "named                17\n",
      "ps                   15\n",
      "sendmail             14\n",
      "rootkit              13\n",
      "xterm                13\n",
      "teardrop             12\n",
      "xlock                 9\n",
      "land                  7\n",
      "xsnoop                4\n",
      "ftp_write             3\n",
      "perl                  2\n",
      "phf                   2\n",
      "loadmodule            2\n",
      "sqlattack             2\n",
      "udpstorm              2\n",
      "worm                  2\n",
      "imap                  1\n",
      "Name: label, dtype: int64\n",
      "Training set:\n",
      "Feature 'protocol_type' has 3 categories\n",
      "Feature 'service' has 70 categories\n",
      "Feature 'flag' has 11 categories\n",
      "Feature 'label' has 23 categories\n",
      "\n",
      "Distribution of categories in service:\n",
      "http        40338\n",
      "private     21853\n",
      "domain_u     9043\n",
      "smtp         7313\n",
      "ftp_data     6860\n",
      "Name: service, dtype: int64\n",
      "Test set:\n",
      "Feature 'protocol_type' has 3 categories\n",
      "Feature 'service' has 64 categories\n",
      "Feature 'flag' has 11 categories\n",
      "Feature 'label' has 38 categories\n",
      "['Protocol_type_icmp', 'Protocol_type_tcp', 'Protocol_type_udp', 'service_IRC', 'service_X11', 'service_Z39_50', 'service_aol', 'service_auth', 'service_bgp', 'service_courier', 'service_csnet_ns', 'service_ctf', 'service_daytime', 'service_discard', 'service_domain', 'service_domain_u', 'service_echo', 'service_eco_i', 'service_ecr_i', 'service_efs', 'service_exec', 'service_finger', 'service_ftp', 'service_ftp_data', 'service_gopher', 'service_harvest', 'service_hostnames', 'service_http', 'service_http_2784', 'service_http_443', 'service_http_8001', 'service_imap4', 'service_iso_tsap', 'service_klogin', 'service_kshell', 'service_ldap', 'service_link', 'service_login', 'service_mtp', 'service_name', 'service_netbios_dgm', 'service_netbios_ns', 'service_netbios_ssn', 'service_netstat', 'service_nnsp', 'service_nntp', 'service_ntp_u', 'service_other', 'service_pm_dump', 'service_pop_2', 'service_pop_3', 'service_printer', 'service_private', 'service_red_i', 'service_remote_job', 'service_rje', 'service_shell', 'service_smtp', 'service_sql_net', 'service_ssh', 'service_sunrpc', 'service_supdup', 'service_systat', 'service_telnet', 'service_tftp_u', 'service_tim_i', 'service_time', 'service_urh_i', 'service_urp_i', 'service_uucp', 'service_uucp_path', 'service_vmnet', 'service_whois', 'flag_OTH', 'flag_REJ', 'flag_RSTO', 'flag_RSTOS0', 'flag_RSTR', 'flag_S0', 'flag_S1', 'flag_S2', 'flag_S3', 'flag_SF', 'flag_SH']\n",
      "   protocol_type  service  flag\n",
      "0              1       20     9\n",
      "1              2       44     9\n",
      "2              1       49     5\n",
      "3              1       24     9\n",
      "4              1       24     9\n",
      "(125973, 123)\n",
      "(22544, 123)\n",
      "0    0\n",
      "1    0\n",
      "2    1\n",
      "3    0\n",
      "4    0\n",
      "Name: label, dtype: int64\n",
      "X_train has shape: (125973, 122) \n",
      "y_train has shape: (125973, 1)\n",
      "X_test has shape: (22544, 122) \n",
      "y_test has shape: (22544, 1)\n",
      "Counter({0: 67343, 1: 45927, 2: 11656, 3: 995, 4: 52})\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# shape, this gives the dimensions of the dataset\n",
    "print('Dimensions of the Training set:',df.shape)\n",
    "print('Dimensions of the Test set:',df_test.shape)\n",
    "\n",
    "\n",
    "df.drop(['difficulty'],axis=1,inplace=True)\n",
    "df_test.drop(['difficulty'],axis=1,inplace=True)\n",
    "\n",
    "print('Label distribution Training set:')\n",
    "print(df['label'].value_counts())\n",
    "print()\n",
    "print('Label distribution Test set:')\n",
    "print(df_test['label'].value_counts())\n",
    "\n",
    "# colums that are categorical and not binary yet: protocol_type (column 2), service (column 3), flag (column 4).\n",
    "# explore categorical features\n",
    "print('Training set:')\n",
    "for col_name in df.columns:\n",
    "    if df[col_name].dtypes == 'object' :\n",
    "        unique_cat = len(df[col_name].unique())\n",
    "        print(\"Feature '{col_name}' has {unique_cat} categories\".format(col_name=col_name, unique_cat=unique_cat))\n",
    "\n",
    "#see how distributed the feature service is, it is evenly distributed and therefore we need to make dummies for all.\n",
    "print()\n",
    "print('Distribution of categories in service:')\n",
    "print(df['service'].value_counts().sort_values(ascending=False).head())\n",
    "\n",
    "# Test set\n",
    "print('Test set:')\n",
    "for col_name in df_test.columns:\n",
    "    if df_test[col_name].dtypes == 'object' :\n",
    "        unique_cat = len(df_test[col_name].unique())\n",
    "        print(\"Feature '{col_name}' has {unique_cat} categories\".format(col_name=col_name, unique_cat=unique_cat))\n",
    "\n",
    "\n",
    "categorical_columns=['protocol_type', 'service', 'flag']\n",
    "# insert code to get a list of categorical columns into a variable, categorical_columns\n",
    "categorical_columns=['protocol_type', 'service', 'flag'] \n",
    " # Get the categorical values into a 2D numpy array\n",
    "df_categorical_values = df[categorical_columns]\n",
    "testdf_categorical_values = df_test[categorical_columns]\n",
    "df_categorical_values.head()\n",
    "\n",
    "\n",
    "# protocol type\n",
    "unique_protocol=sorted(df.protocol_type.unique())\n",
    "string1 = 'Protocol_type_'\n",
    "unique_protocol2=[string1 + x for x in unique_protocol]\n",
    "# service\n",
    "unique_service=sorted(df.service.unique())\n",
    "string2 = 'service_'\n",
    "unique_service2=[string2 + x for x in unique_service]\n",
    "# flag\n",
    "unique_flag=sorted(df.flag.unique())\n",
    "string3 = 'flag_'\n",
    "unique_flag2=[string3 + x for x in unique_flag]\n",
    "# put together\n",
    "dumcols=unique_protocol2 + unique_service2 + unique_flag2\n",
    "print(dumcols)\n",
    "\n",
    "#do same for test set\n",
    "unique_service_test=sorted(df_test.service.unique())\n",
    "unique_service2_test=[string2 + x for x in unique_service_test]\n",
    "testdumcols=unique_protocol2 + unique_service2_test + unique_flag2\n",
    "\n",
    "df_categorical_values_enc=df_categorical_values.apply(LabelEncoder().fit_transform)\n",
    "print(df_categorical_values_enc.head())\n",
    "# test set\n",
    "testdf_categorical_values_enc=testdf_categorical_values.apply(LabelEncoder().fit_transform)\n",
    "\n",
    "enc = OneHotEncoder()\n",
    "df_categorical_values_encenc = enc.fit_transform(df_categorical_values_enc)\n",
    "df_cat_data = pd.DataFrame(df_categorical_values_encenc.toarray(),columns=dumcols)\n",
    "# test set\n",
    "testdf_categorical_values_encenc = enc.fit_transform(testdf_categorical_values_enc)\n",
    "testdf_cat_data = pd.DataFrame(testdf_categorical_values_encenc.toarray(),columns=testdumcols)\n",
    "\n",
    "df_cat_data.head()\n",
    "\n",
    "trainservice=df['service'].tolist()\n",
    "testservice= df_test['service'].tolist()\n",
    "difference=list(set(trainservice) - set(testservice))\n",
    "string = 'service_'\n",
    "difference=[string + x for x in difference]\n",
    "difference\n",
    "\n",
    "for col in difference:\n",
    "    testdf_cat_data[col] = 0\n",
    "\n",
    "testdf_cat_data.shape\n",
    "\n",
    "newdf=df.join(df_cat_data)\n",
    "newdf.drop('flag', axis=1, inplace=True)\n",
    "newdf.drop('protocol_type', axis=1, inplace=True)\n",
    "newdf.drop('service', axis=1, inplace=True)\n",
    "# test data\n",
    "newdf_test=df_test.join(testdf_cat_data)\n",
    "newdf_test.drop('flag', axis=1, inplace=True)\n",
    "newdf_test.drop('protocol_type', axis=1, inplace=True)\n",
    "newdf_test.drop('service', axis=1, inplace=True)\n",
    "print(newdf.shape)\n",
    "print(newdf_test.shape)\n",
    "\n",
    "\n",
    "# take label column\n",
    "labeldf=newdf['label']\n",
    "labeldf_test=newdf_test['label']\n",
    "# change the label column\n",
    "newlabeldf=labeldf.replace({ 'normal' : 0, 'neptune' : 1 ,'back': 1, 'land': 1, 'pod': 1, 'smurf': 1, 'teardrop': 1,'mailbomb': 1, 'apache2': 1, 'processtable': 1, 'udpstorm': 1, 'worm': 1,\n",
    "                           'ipsweep' : 2,'nmap' : 2,'portsweep' : 2,'satan' : 2,'mscan' : 2,'saint' : 2\n",
    "                           ,'ftp_write': 3,'guess_passwd': 3,'imap': 3,'multihop': 3,'phf': 3,'spy': 3,'warezclient': 3,'warezmaster': 3,'sendmail': 3,'named': 3,'snmpgetattack': 3,'snmpguess': 3,'xlock': 3,'xsnoop': 3,'httptunnel': 3,\n",
    "                           'buffer_overflow': 4,'loadmodule': 4,'perl': 4,'rootkit': 4,'ps': 4,'sqlattack': 4,'xterm': 4})\n",
    "newlabeldf_test=labeldf_test.replace({ 'normal' : 0, 'neptune' : 1 ,'back': 1, 'land': 1, 'pod': 1, 'smurf': 1, 'teardrop': 1,'mailbomb': 1, 'apache2': 1, 'processtable': 1, 'udpstorm': 1, 'worm': 1,\n",
    "                           'ipsweep' : 2,'nmap' : 2,'portsweep' : 2,'satan' : 2,'mscan' : 2,'saint' : 2\n",
    "                           ,'ftp_write': 3,'guess_passwd': 3,'imap': 3,'multihop': 3,'phf': 3,'spy': 3,'warezclient': 3,'warezmaster': 3,'sendmail': 3,'named': 3,'snmpgetattack': 3,'snmpguess': 3,'xlock': 3,'xsnoop': 3,'httptunnel': 3,\n",
    "                           'buffer_overflow': 4,'loadmodule': 4,'perl': 4,'rootkit': 4,'ps': 4,'sqlattack': 4,'xterm': 4})\n",
    "# put the new label column back\n",
    "newdf['label'] = newlabeldf\n",
    "newdf_test['label'] = newlabeldf_test\n",
    "print(newdf['label'].head())\n",
    "\n",
    "\n",
    "# Specify your selected features. Note that you'll need to modify this list according to your final processed dataframe\n",
    "#Uncomment the below lines to use these top 20 features from shap analysis\n",
    "#selected_features = [\"root_shell\",\"service_telnet\",\"num_shells\",\"service_uucp\",\"dst_host_same_src_port_rate\"\n",
    "#                     ,\"dst_host_rerror_rate\",\"dst_host_srv_serror_rate\",\"dst_host_srv_count\",\"service_private\",\"logged_in\",\n",
    "#                    \"dst_host_serror_rate\",\"serror_rate\",\"srv_serror_rate\",\"flag_S0\",\"diff_srv_rate\",\"dst_host_srv_diff_host_rate\",\"num_file_creations\",\"flag_RSTR\"#,\"dst_host_same_srv_rate\",\"service_Idap\",\"label\"]\n",
    "                     \n",
    "    \n",
    "# Select those features from your dataframe\n",
    "#newdf = newdf[selected_features]\n",
    "#newdf_test = newdf_test[selected_features]\n",
    "\n",
    "# Now your dataframe only contains your selected features.\n",
    "\n",
    "# creating a dataframe with multi-class labels (Dos,Probe,R2L,U2R,normal)\n",
    "multi_data = newdf.copy()\n",
    "multi_label = pd.DataFrame(multi_data.label)\n",
    "\n",
    "multi_data_test=newdf_test.copy()\n",
    "multi_label_test = pd.DataFrame(multi_data_test.label)\n",
    "\n",
    "\n",
    "# using standard scaler for normalizing\n",
    "std_scaler = StandardScaler()\n",
    "def standardization(df,col):\n",
    "    for i in col:\n",
    "        arr = df[i]\n",
    "        arr = np.array(arr)\n",
    "        df[i] = std_scaler.fit_transform(arr.reshape(len(arr),1))\n",
    "    return df\n",
    "\n",
    "numeric_col = multi_data.select_dtypes(include='number').columns\n",
    "data = standardization(multi_data,numeric_col)\n",
    "numeric_col_test = multi_data_test.select_dtypes(include='number').columns\n",
    "data_test = standardization(multi_data_test,numeric_col_test)\n",
    "\n",
    "# label encoding (0,1,2,3,4) multi-class labels (Dos,normal,Probe,R2L,U2R)\n",
    "le2 = preprocessing.LabelEncoder()\n",
    "le2_test = preprocessing.LabelEncoder()\n",
    "enc_label = multi_label.apply(le2.fit_transform)\n",
    "enc_label_test = multi_label_test.apply(le2_test.fit_transform)\n",
    "multi_data = multi_data.copy()\n",
    "multi_data_test = multi_data_test.copy()\n",
    "\n",
    "multi_data['intrusion'] = enc_label\n",
    "multi_data_test['intrusion'] = enc_label_test\n",
    "\n",
    "#y_mul = multi_data['intrusion']\n",
    "multi_data\n",
    "multi_data_test\n",
    "\n",
    "multi_data.drop(labels= [ 'label'], axis=1, inplace=True)\n",
    "multi_data\n",
    "multi_data_test.drop(labels= [ 'label'], axis=1, inplace=True)\n",
    "multi_data_test\n",
    "\n",
    "y_train_multi= multi_data[['intrusion']]\n",
    "X_train_multi= multi_data.drop(labels=['intrusion'], axis=1)\n",
    "\n",
    "print('X_train has shape:',X_train_multi.shape,'\\ny_train has shape:',y_train_multi.shape)\n",
    "\n",
    "y_test_multi= multi_data_test[['intrusion']]\n",
    "X_test_multi= multi_data_test.drop(labels=['intrusion'], axis=1)\n",
    "\n",
    "print('X_test has shape:',X_test_multi.shape,'\\ny_test has shape:',y_test_multi.shape)\n",
    "\n",
    "label_counts = Counter(y_train_multi['intrusion'])\n",
    "print(label_counts)\n",
    "\n",
    "y_train_multi = LabelBinarizer().fit_transform(y_train_multi)\n",
    "\n",
    "y_test_multi = LabelBinarizer().fit_transform(y_test_multi)\n",
    "\n",
    "Y_train=y_train_multi.copy()\n",
    "X_train=X_train_multi.copy()\n",
    "\n",
    "Y_test=y_test_multi.copy()\n",
    "X_test=X_test_multi.copy()\n",
    "\n",
    "ros = RandomOverSampler(sampling_strategy='minority', random_state=100)\n",
    "\n",
    "X_train, Y_train = ros.fit_resample(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "single_class_train = np.argmax(y_train_multi, axis=1)\n",
    "single_class_test = np.argmax(y_test_multi, axis=1)\n",
    "\n",
    "df1 = X_train_multi.assign(Label = single_class_train)\n",
    "df2 =  X_test_multi.assign(Label = single_class_test)\n",
    "\n",
    "frames = [df1,  df2]\n",
    "\n",
    "df = pd.concat(frames,ignore_index=True)\n",
    "\n",
    "df = df.sample(15000)\n",
    "\n",
    "\n",
    "df_fs = df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.pop('Label')\n",
    "X = df\n",
    "df = X.assign(Label = y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection Methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set feature selection = 1 to use Information gain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "if feature_selection_bit == 1:\n",
    "\n",
    "    # Compute information gain using mutual information\n",
    "    importances = mutual_info_classif(X, y)\n",
    "\n",
    "    feat_importances = pd.Series(importances, df.columns[0:len(df.columns)-1])\n",
    "    # feat_importances.plot(kind='barh', color = 'teal')\n",
    "        \n",
    "    feat_importances_sorted = feat_importances.sort_values( ascending=False)\n",
    "\n",
    "    # Print or use the sorted DataFrame\n",
    "    print(feat_importances_sorted)\n",
    "    # feat_importances_sorted.plot(kind='barh', color = 'teal')\n",
    "    # feat_importances_sorted\n",
    "    top_features = feat_importances_sorted.nlargest(10)\n",
    "    top_feature_names = top_features.index.tolist()\n",
    "\n",
    "    print(\"Top 10 feature names:\")\n",
    "    print(top_feature_names)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if feature_selection_bit == 1:\n",
    "    # USE XAI feature selection from last work https://www.mdpi.com/2076-3417/14/10/4170\n",
    "    feature_selection = [\n",
    "                        'dst_host_same_srv_rate',\n",
    "                        'dst_host_srv_count',\n",
    "                        'dst_host_same_src_port_rate',\n",
    "                        'logged_in',\n",
    "                        'dst_host_serror_rate',\n",
    "                        'count',\n",
    "                        'srv_count',\n",
    "                        'dst_host_rerror_rate',\n",
    "                        'Label'\n",
    "                        ]\n",
    "    \n",
    "    # to Use information gain uncomment line below\n",
    "    # feature_selection = top_feature_names\n",
    "    \n",
    "    df_og = df\n",
    "    df = df[feature_selection]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Create One vs ALL labels that will be needed later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# y = df.pop('Label')\n",
    "# X = df\n",
    "\n",
    "y1, y2 = pd.factorize(y)\n",
    "\n",
    "y_0 = pd.DataFrame(y1)\n",
    "y_1 = pd.DataFrame(y1)\n",
    "y_2 = pd.DataFrame(y1)\n",
    "y_3 = pd.DataFrame(y1)\n",
    "y_4 = pd.DataFrame(y1)\n",
    "\n",
    "\n",
    "# y_0 = y_0.replace(0, 0)\n",
    "# y_0 = y_0.replace(1, 1)\n",
    "y_0 = y_0.replace(2, 1)\n",
    "y_0 = y_0.replace(3, 1)\n",
    "y_0 = y_0.replace(4, 1)\n",
    "\n",
    "\n",
    "y_1 = y_1.replace(1, 999)\n",
    "y_1 = y_1.replace(0, 1)\n",
    "# y_1 = y_1.replace(1, 0)\n",
    "y_1 = y_1.replace(2, 1)\n",
    "y_1 = y_1.replace(3, 1)\n",
    "y_1 = y_1.replace(4, 1)\n",
    "y_1 = y_1.replace(999, 1)\n",
    "\n",
    "\n",
    "y_2 = y_2.replace(0, 1)\n",
    "y_2 = y_2.replace(1, 1)\n",
    "y_2 = y_2.replace(2, 0)\n",
    "y_2 = y_2.replace(3, 1)\n",
    "y_2 = y_2.replace(4, 1)\n",
    "\n",
    "\n",
    "y_3 = y_3.replace(0, 1)\n",
    "# y_3 = y_3.replace(1, 1)\n",
    "y_3 = y_3.replace(2, 1)\n",
    "y_3 = y_3.replace(3, 0)\n",
    "y_3 = y_3.replace(4, 1)\n",
    "\n",
    "\n",
    "y_4 = y_4.replace(0, 1)\n",
    "# y_4 = y_4.replace(1, 1)\n",
    "y_4 = y_4.replace(2, 1)\n",
    "y_4 = y_4.replace(3, 1)\n",
    "y_4 = y_4.replace(4, 0)\n",
    "\n",
    "\n",
    "\n",
    "df = df.assign(Label = y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Split dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "split = 0.7\n",
    "\n",
    "X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({0: 7804, 1: 5334, 2: 1415, 3: 438, 4: 9})\n"
     ]
    }
   ],
   "source": [
    "\n",
    "label_counts2 = Counter(y)\n",
    "print(label_counts2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>...</th>\n",
       "      <th>flag_REJ</th>\n",
       "      <th>flag_RSTO</th>\n",
       "      <th>flag_RSTOS0</th>\n",
       "      <th>flag_RSTR</th>\n",
       "      <th>flag_S0</th>\n",
       "      <th>flag_S1</th>\n",
       "      <th>flag_S2</th>\n",
       "      <th>flag_S3</th>\n",
       "      <th>flag_SF</th>\n",
       "      <th>flag_SH</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12261</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007723</td>\n",
       "      <td>-0.004849</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>-0.618438</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>0.825150</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95229</th>\n",
       "      <td>-0.109865</td>\n",
       "      <td>-0.007536</td>\n",
       "      <td>-0.004836</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>-0.618438</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>0.825150</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25115</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007727</td>\n",
       "      <td>-0.004835</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>-0.618438</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>0.825150</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105746</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007762</td>\n",
       "      <td>-0.004919</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>-0.809262</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>1.616978</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>-1.211901</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26007</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007762</td>\n",
       "      <td>-0.004919</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>-0.809262</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>1.616978</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>-1.211901</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138967</th>\n",
       "      <td>-0.155534</td>\n",
       "      <td>-0.019856</td>\n",
       "      <td>-0.096896</td>\n",
       "      <td>-0.017624</td>\n",
       "      <td>-0.059104</td>\n",
       "      <td>-0.019459</td>\n",
       "      <td>-0.113521</td>\n",
       "      <td>-0.143999</td>\n",
       "      <td>-0.890373</td>\n",
       "      <td>-0.016494</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.453815</td>\n",
       "      <td>-0.18843</td>\n",
       "      <td>-0.009419</td>\n",
       "      <td>-0.174880</td>\n",
       "      <td>-0.313124</td>\n",
       "      <td>-0.030535</td>\n",
       "      <td>-0.025803</td>\n",
       "      <td>-0.105681</td>\n",
       "      <td>0.718027</td>\n",
       "      <td>-0.056997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108216</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007715</td>\n",
       "      <td>-0.003932</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>-0.618438</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>0.825150</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73259</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007726</td>\n",
       "      <td>-0.003903</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>-0.618438</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>0.825150</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28009</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007762</td>\n",
       "      <td>-0.004919</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>-0.809262</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>1.616978</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>-1.211901</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16436</th>\n",
       "      <td>-0.110249</td>\n",
       "      <td>-0.007762</td>\n",
       "      <td>-0.004919</td>\n",
       "      <td>-0.014089</td>\n",
       "      <td>-0.089486</td>\n",
       "      <td>-0.007736</td>\n",
       "      <td>-0.095076</td>\n",
       "      <td>-0.027023</td>\n",
       "      <td>-0.809262</td>\n",
       "      <td>-0.011664</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.312889</td>\n",
       "      <td>-0.11205</td>\n",
       "      <td>-0.028606</td>\n",
       "      <td>-0.139982</td>\n",
       "      <td>1.616978</td>\n",
       "      <td>-0.053906</td>\n",
       "      <td>-0.031767</td>\n",
       "      <td>-0.019726</td>\n",
       "      <td>-1.211901</td>\n",
       "      <td>-0.046432</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10500 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        duration  src_bytes  dst_bytes      land  wrong_fragment    urgent  \\\n",
       "12261  -0.110249  -0.007723  -0.004849 -0.014089       -0.089486 -0.007736   \n",
       "95229  -0.109865  -0.007536  -0.004836 -0.014089       -0.089486 -0.007736   \n",
       "25115  -0.110249  -0.007727  -0.004835 -0.014089       -0.089486 -0.007736   \n",
       "105746 -0.110249  -0.007762  -0.004919 -0.014089       -0.089486 -0.007736   \n",
       "26007  -0.110249  -0.007762  -0.004919 -0.014089       -0.089486 -0.007736   \n",
       "...          ...        ...        ...       ...             ...       ...   \n",
       "138967 -0.155534  -0.019856  -0.096896 -0.017624       -0.059104 -0.019459   \n",
       "108216 -0.110249  -0.007715  -0.003932 -0.014089       -0.089486 -0.007736   \n",
       "73259  -0.110249  -0.007726  -0.003903 -0.014089       -0.089486 -0.007736   \n",
       "28009  -0.110249  -0.007762  -0.004919 -0.014089       -0.089486 -0.007736   \n",
       "16436  -0.110249  -0.007762  -0.004919 -0.014089       -0.089486 -0.007736   \n",
       "\n",
       "             hot  num_failed_logins  logged_in  num_compromised  ...  \\\n",
       "12261  -0.095076          -0.027023   1.235694        -0.011664  ...   \n",
       "95229  -0.095076          -0.027023   1.235694        -0.011664  ...   \n",
       "25115  -0.095076          -0.027023   1.235694        -0.011664  ...   \n",
       "105746 -0.095076          -0.027023  -0.809262        -0.011664  ...   \n",
       "26007  -0.095076          -0.027023  -0.809262        -0.011664  ...   \n",
       "...          ...                ...        ...              ...  ...   \n",
       "138967 -0.113521          -0.143999  -0.890373        -0.016494  ...   \n",
       "108216 -0.095076          -0.027023   1.235694        -0.011664  ...   \n",
       "73259  -0.095076          -0.027023   1.235694        -0.011664  ...   \n",
       "28009  -0.095076          -0.027023  -0.809262        -0.011664  ...   \n",
       "16436  -0.095076          -0.027023  -0.809262        -0.011664  ...   \n",
       "\n",
       "        flag_REJ  flag_RSTO  flag_RSTOS0  flag_RSTR   flag_S0   flag_S1  \\\n",
       "12261  -0.312889   -0.11205    -0.028606  -0.139982 -0.618438 -0.053906   \n",
       "95229  -0.312889   -0.11205    -0.028606  -0.139982 -0.618438 -0.053906   \n",
       "25115  -0.312889   -0.11205    -0.028606  -0.139982 -0.618438 -0.053906   \n",
       "105746 -0.312889   -0.11205    -0.028606  -0.139982  1.616978 -0.053906   \n",
       "26007  -0.312889   -0.11205    -0.028606  -0.139982  1.616978 -0.053906   \n",
       "...          ...        ...          ...        ...       ...       ...   \n",
       "138967 -0.453815   -0.18843    -0.009419  -0.174880 -0.313124 -0.030535   \n",
       "108216 -0.312889   -0.11205    -0.028606  -0.139982 -0.618438 -0.053906   \n",
       "73259  -0.312889   -0.11205    -0.028606  -0.139982 -0.618438 -0.053906   \n",
       "28009  -0.312889   -0.11205    -0.028606  -0.139982  1.616978 -0.053906   \n",
       "16436  -0.312889   -0.11205    -0.028606  -0.139982  1.616978 -0.053906   \n",
       "\n",
       "         flag_S2   flag_S3   flag_SF   flag_SH  \n",
       "12261  -0.031767 -0.019726  0.825150 -0.046432  \n",
       "95229  -0.031767 -0.019726  0.825150 -0.046432  \n",
       "25115  -0.031767 -0.019726  0.825150 -0.046432  \n",
       "105746 -0.031767 -0.019726 -1.211901 -0.046432  \n",
       "26007  -0.031767 -0.019726 -1.211901 -0.046432  \n",
       "...          ...       ...       ...       ...  \n",
       "138967 -0.025803 -0.105681  0.718027 -0.056997  \n",
       "108216 -0.031767 -0.019726  0.825150 -0.046432  \n",
       "73259  -0.031767 -0.019726  0.825150 -0.046432  \n",
       "28009  -0.031767 -0.019726 -1.211901 -0.046432  \n",
       "16436  -0.031767 -0.019726 -1.211901 -0.046432  \n",
       "\n",
       "[10500 rows x 122 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12261     0\n",
       "95229     0\n",
       "25115     0\n",
       "105746    1\n",
       "26007     1\n",
       "         ..\n",
       "138967    1\n",
       "108216    0\n",
       "73259     0\n",
       "28009     1\n",
       "16436     1\n",
       "Name: Label, Length: 10500, dtype: int64"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LEVEL 0 - Weak models - Base Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_12 (Dense)             (None, 3)                 369       \n",
      "_________________________________________________________________\n",
      "dropout_10 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_11 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_12 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 5)                 20        \n",
      "=================================================================\n",
      "Total params: 437\n",
      "Trainable params: 437\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------START of WEAK LEARNERS (BASE MODELS) - STACK 00 -----------------', file = f)\n",
    "\n",
    "#Defining Basemodels\n",
    "\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "lgbm = LGBMClassifier()\n",
    "knn_clf=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "# SVM\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "\n",
    "#DNN\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train.shape[1]\n",
    "\n",
    "dnn = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "dnn.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training SVM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n",
      "66/66 [==============================] - 1s 4ms/step - loss: 1.5626 - accuracy: 0.4537 - val_loss: 1.3482 - val_accuracy: 0.8100\n",
      "Epoch 2/100\n",
      "66/66 [==============================] - 0s 3ms/step - loss: 1.2600 - accuracy: 0.6408 - val_loss: 0.7604 - val_accuracy: 0.8514\n",
      "Epoch 3/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.9579 - accuracy: 0.7426 - val_loss: 0.4993 - val_accuracy: 0.8595\n",
      "Epoch 4/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.8161 - accuracy: 0.7748 - val_loss: 0.4345 - val_accuracy: 0.8605\n",
      "Epoch 5/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.7331 - accuracy: 0.7944 - val_loss: 0.4167 - val_accuracy: 0.8629\n",
      "Epoch 6/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.7038 - accuracy: 0.8013 - val_loss: 0.4062 - val_accuracy: 0.8633\n",
      "Epoch 7/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.6775 - accuracy: 0.8064 - val_loss: 0.3976 - val_accuracy: 0.8614\n",
      "Epoch 8/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.6398 - accuracy: 0.8108 - val_loss: 0.3906 - val_accuracy: 0.8624\n",
      "Epoch 9/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.6338 - accuracy: 0.8123 - val_loss: 0.3832 - val_accuracy: 0.8605\n",
      "Epoch 10/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5914 - accuracy: 0.8170 - val_loss: 0.3792 - val_accuracy: 0.8610\n",
      "Epoch 11/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5899 - accuracy: 0.8180 - val_loss: 0.3716 - val_accuracy: 0.8610\n",
      "Epoch 12/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5998 - accuracy: 0.8177 - val_loss: 0.3647 - val_accuracy: 0.8624\n",
      "Epoch 13/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5521 - accuracy: 0.8205 - val_loss: 0.3633 - val_accuracy: 0.8624\n",
      "Epoch 14/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5483 - accuracy: 0.8233 - val_loss: 0.3608 - val_accuracy: 0.8624\n",
      "Epoch 15/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5526 - accuracy: 0.8230 - val_loss: 0.3571 - val_accuracy: 0.8629\n",
      "Epoch 16/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5505 - accuracy: 0.8248 - val_loss: 0.3535 - val_accuracy: 0.8633\n"
     ]
    }
   ],
   "source": [
    "#Training Basemodels\n",
    "n_splits = 5  # You can adjust the number of folds as needed\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Training Model')\n",
    "with open(output_file_name, \"a\") as f: print('Training weak models - level 0', file = f)\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1 and load_model_ada == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    ada_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_rf == 1 and load_model_rf == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf = rf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    rf_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf, 'rf_base_model.joblib')\n",
    "\n",
    "if use_model_svm == 1 and load_model_svm == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "\n",
    "    start = time.time()\n",
    "    clf.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "    # clf.score(X_train, y_train)\n",
    "    svm_tr_time_taken= time_taken = end - start\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(clf, 'svm_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_knn == 1 and load_model_knn == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    knn_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf, 'knn_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_lgbm == 1 and load_model_lgbm == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    lgbm_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_base_model.joblib')\n",
    "\n",
    "if use_model_mlp == 1 and load_model_mlp == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    mlp_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_dnn == 1 and load_model_dnn == 0:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    dnn.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "\n",
    "    dnn_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not implemented, the idea was to save and load the model if you did not want to run it again. \n",
    "\n",
    "if load_model_ada == 1:\n",
    "    ada = joblib.load('ada_base_model.joblib')\n",
    "\n",
    "if load_model_svm == 1:\n",
    "    clf =  joblib.load('svm_base_model.joblib')\n",
    "\n",
    "if load_model_dnn == 1:\n",
    "    dnn = load_model(\"DNN_base_model.h5\")\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf = joblib.load('knn_base_model.joblib')\n",
    "\n",
    "if load_model_mlp == 1:\n",
    "    MLP = joblib.load('mlp_base_model.joblib')\n",
    "\n",
    "if load_model_rf == 1:\n",
    "    rf = joblib.load('rf_base_model.joblib')\n",
    "\n",
    "if load_model_lgbm == 1:\n",
    "    lgbm = joblib.load('lgbm_base_model.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base leaners predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction SVM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction DNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction KNN\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "\n",
    "with open(output_file_name, \"a\") as f: print('Generating Predictions', file = f)\n",
    "\n",
    "if use_model_rf == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf = rf.predict(X_test)\n",
    "    preds_rf_prob = rf.predict_proba(X_test)\n",
    "    end = time.time()\n",
    "    rf_pr_time_taken=  time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_svm == 1:\n",
    "\n",
    "    print('Prediction SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "    start = time.time()\n",
    "    preds_svm = clf.predict(X_test)\n",
    "    # preds_svm_prob = clf.predict_proba(X_test)\n",
    "\n",
    "    #Since SVM does not deal with prob by nature we use a meta learner\n",
    "    # https://stackoverflow.com/questions/55250963/how-to-get-probabilities-for-sgdclassifier-linearsvm\n",
    "\n",
    "    model = CalibratedClassifierCV(clf)\n",
    "\n",
    "    model.fit(X, y)\n",
    "    preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "    end = time.time()\n",
    "    svm_pr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm = lgbm.predict(X_test)\n",
    "    preds_lgbm_prob = lgbm.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    lgbm_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_dnn == 1:\n",
    "\n",
    "    print('Prediction DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #DNN\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn.predict(X_test)\n",
    "    preds_dnn_prob = pred_dnn\n",
    "    preds_dnn = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    dnn_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada = ada.predict(X_test)\n",
    "    preds_ada_prob = ada.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    ada_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_mlp == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test)\n",
    "    preds_mlp_prob = y_pred\n",
    "    preds_mlp = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    mlp_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf.predict(X_test)\n",
    "    preds_knn_prob =knn_clf.predict_proba(X_test)\n",
    "\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    knn_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_3 = np.argmax(preds_svm_prob,axis = 1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### METRICS - Base Learners"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2    3    4\n",
      "0  2363.0     1.0    3.0  0.0  0.0\n",
      "1    43.0  1551.0    9.0  0.0  0.0\n",
      "2    33.0    10.0  363.0  0.0  0.0\n",
      "3   109.0     0.0    6.0  8.0  0.0\n",
      "4     1.0     0.0    0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9522222222222222\n",
      "Precision total:  0.7745487719830721\n",
      "Recall total:  0.5850000481965352\n",
      "F1 total:  0.597214670613797\n",
      "BACC total:  0.5850000481965352\n",
      "MCC total:  0.9186776002432332\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#RF\n",
    "if use_model_rf == 1:\n",
    "    # start = time.time()\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('RF base model', file = f)\n",
    "\n",
    "    pred_label = preds_rf\n",
    "    name = 'rf'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test,rf_pr_time_taken + rf_tr_time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    globals()[f\"{name}_time_00\"] = rf_pr_time_taken + rf_tr_time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1    2    3    4\n",
      "0  2331.0    36.0  0.0  0.0  0.0\n",
      "1    62.0  1541.0  0.0  0.0  0.0\n",
      "2   282.0   124.0  0.0  0.0  0.0\n",
      "3   118.0     5.0  0.0  0.0  0.0\n",
      "4     1.0     0.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8604444444444445\n",
      "Precision total:  0.3475140583447532\n",
      "Recall total:  0.38922267895984003\n",
      "F1 total:  0.3669425051275786\n",
      "BACC total:  0.38922267895984003\n",
      "MCC total:  0.7582984984810872\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "if use_model_dnn == 1:\n",
    "    start = time.time()\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('DNN base model', file = f)\n",
    "\n",
    "\n",
    "    pred_label = preds_dnn\n",
    "    name = 'dnn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test, dnn_pr_time_taken + dnn_tr_time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = dnn_pr_time_taken + dnn_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0      1       2    3    4\n",
      "0  2122.0  162.0    78.0  5.0  0.0\n",
      "1   150.0   65.0  1384.0  4.0  0.0\n",
      "2   155.0    6.0   242.0  3.0  0.0\n",
      "3   109.0    7.0     7.0  0.0  0.0\n",
      "4     1.0    0.0     0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.5397777777777778\n",
      "Precision total:  0.24973841173618946\n",
      "Recall total:  0.30662030712139987\n",
      "F1 total:  0.2329157135863175\n",
      "BACC total:  0.30662030712139987\n",
      "MCC total:  0.33871380906463655\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#ADA\n",
    "if use_model_ada == 1:\n",
    "    start = time.time()\n",
    "    \n",
    "    pred_label = preds_ada\n",
    "    name = 'ada'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test, ada_pr_time_taken + ada_tr_time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = ada_pr_time_taken + ada_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2     3    4\n",
      "0  2290.0    28.0   23.0  26.0  0.0\n",
      "1    25.0  1577.0    1.0   0.0  0.0\n",
      "2    29.0    16.0  359.0   2.0  0.0\n",
      "3    33.0     2.0    0.0  88.0  0.0\n",
      "4     0.0     0.0    0.0   0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9588888888888889\n",
      "Precision total:  0.9262028343084049\n",
      "Recall total:  0.9101866779825443\n",
      "F1 total:  0.9179051407788913\n",
      "BACC total:  0.9101866779825443\n",
      "MCC total:  0.9298092147834897\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "if use_model_svm == 1:\n",
    "    start = time.time()\n",
    "\n",
    "    pred_label = preds_svm\n",
    "    name = 'svm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test,svm_pr_time_taken + svm_tr_time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = svm_pr_time_taken + svm_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2332.0    15.0   11.0    9.0  0.0\n",
      "1     2.0  1598.0    2.0    1.0  0.0\n",
      "2    12.0    17.0  377.0    0.0  0.0\n",
      "3    12.0     0.0    0.0  111.0  0.0\n",
      "4     1.0     0.0    0.0    0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9817777777777777\n",
      "Precision total:  0.7705889217922\n",
      "Recall total:  0.7626209303206533\n",
      "F1 total:  0.7665017702069287\n",
      "BACC total:  0.7626209303206533\n",
      "MCC total:  0.9689838564473047\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "if use_model_knn == 1:\n",
    "    start = time.time()\n",
    "    pred_label = preds_knn\n",
    "    name = 'knn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test,knn_pr_time_taken + knn_tr_time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = knn_pr_time_taken + knn_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2339.0     8.0    5.0   15.0  0.0\n",
      "1     4.0  1599.0    0.0    0.0  0.0\n",
      "2    12.0     5.0  386.0    3.0  0.0\n",
      "3     9.0     0.0    0.0  113.0  1.0\n",
      "4     0.0     0.0    0.0    0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9862222222222222\n",
      "Precision total:  0.8662335767649678\n",
      "Recall total:  0.9710226924322607\n",
      "F1 total:  0.9017144796373003\n",
      "BACC total:  0.9710226924322607\n",
      "MCC total:  0.9765614002651856\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "if use_model_mlp == 1:\n",
    "    start = time.time()\n",
    "    pred_label = preds_mlp\n",
    "    name = 'mlp'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test,mlp_pr_time_taken + mlp_tr_time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = mlp_pr_time_taken + mlp_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2363.0     2.0    1.0    1.0  0.0\n",
      "1     6.0  1596.0    0.0    0.0  1.0\n",
      "2     6.0     2.0  398.0    0.0  0.0\n",
      "3     7.0     0.0    0.0  116.0  0.0\n",
      "4     0.0     0.0    0.0    0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9942222222222222\n",
      "Precision total:  0.8956940470889165\n",
      "Recall total:  0.9834656564678221\n",
      "F1 total:  0.9227751607758353\n",
      "BACC total:  0.9834656564678221\n",
      "MCC total:  0.9901596923430213\n"
     ]
    }
   ],
   "source": [
    "#lgbm\n",
    "start_lgbm = time.time()\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    pred_label = preds_lgbm\n",
    "    name = 'lgbm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test,lgbm_pr_time_taken + lgbm_tr_time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lgbm\n",
    "    globals()[f\"{name}_time_00\"] = lgbm_pr_time_taken + lgbm_tr_time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2342.0     7.0   11.0    6.0  1.0\n",
      "1     8.0  1592.0    3.0    0.0  0.0\n",
      "2    17.0     3.0  386.0    0.0  0.0\n",
      "3     5.0     2.0    1.0  115.0  0.0\n",
      "4     0.0     0.0    0.0    0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9857777777777778\n",
      "Precision total:  0.8785755775571594\n",
      "Recall total:  0.973654847931759\n",
      "F1 total:  0.9094282947512642\n",
      "BACC total:  0.973654847931759\n",
      "MCC total:  0.9757737122479826\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "\n",
    "# Create a Decision Tree Classifier\n",
    "dt_classifier = DecisionTreeClassifier(random_state=42)\n",
    "# Train the classifier on the training data\n",
    "dt_classifier.fit(X_train, y_train)\n",
    "# Make predictions on the test data\n",
    "preds_dt = dt_classifier.predict(X_test)\n",
    "# Evaluate the accuracy of the model\n",
    "preds_dt_prob = dt_classifier.predict_proba(X_test)\n",
    "\n",
    "\n",
    "pred_label = preds_dt\n",
    "name = 'dt'\n",
    "\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test,time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATBOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3205825\ttest: 1.3188172\tbest: 1.3188172 (0)\ttotal: 16.3ms\tremaining: 1.62s\n",
      "10:\tlearn: 0.4449491\ttest: 0.4413229\tbest: 0.4413229 (10)\ttotal: 96.7ms\tremaining: 782ms\n",
      "20:\tlearn: 0.2321835\ttest: 0.2317243\tbest: 0.2317243 (20)\ttotal: 154ms\tremaining: 580ms\n",
      "30:\tlearn: 0.1484856\ttest: 0.1497024\tbest: 0.1497024 (30)\ttotal: 209ms\tremaining: 466ms\n",
      "40:\tlearn: 0.1087867\ttest: 0.1119304\tbest: 0.1119304 (40)\ttotal: 263ms\tremaining: 379ms\n",
      "50:\tlearn: 0.0885619\ttest: 0.0927513\tbest: 0.0927513 (50)\ttotal: 316ms\tremaining: 304ms\n",
      "60:\tlearn: 0.0755467\ttest: 0.0795962\tbest: 0.0795962 (60)\ttotal: 369ms\tremaining: 236ms\n",
      "70:\tlearn: 0.0652197\ttest: 0.0697321\tbest: 0.0697321 (70)\ttotal: 422ms\tremaining: 172ms\n",
      "80:\tlearn: 0.0595860\ttest: 0.0644432\tbest: 0.0644432 (80)\ttotal: 474ms\tremaining: 111ms\n",
      "90:\tlearn: 0.0553427\ttest: 0.0607782\tbest: 0.0607782 (90)\ttotal: 525ms\tremaining: 52ms\n",
      "99:\tlearn: 0.0512405\ttest: 0.0568618\tbest: 0.0568618 (99)\ttotal: 572ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.05686181247\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2353.0     4.0    2.0    8.0  0.0\n",
      "1     9.0  1593.0    1.0    0.0  0.0\n",
      "2    11.0     2.0  393.0    0.0  0.0\n",
      "3    10.0     0.0    3.0  110.0  0.0\n",
      "4     1.0     0.0    0.0    0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9886666666666667\n",
      "Precision total:  0.7800820189850176\n",
      "Recall total:  0.7700272551134686\n",
      "F1 total:  0.7749584037889947\n",
      "BACC total:  0.7700272551134686\n",
      "MCC total:  0.9806802060786379\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "cat_00 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_00.fit(X_train, y_train, eval_set=(X_test, y_test), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_00.predict(X_test)\n",
    "preds_cat_prob = cat_00.predict_proba(X_test)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Catboost base model', file = f)\n",
    "    pred_label = preds_cat\n",
    "    # pred_label = y_pred\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    name = 'cat'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "    globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "start = time.time()\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_00 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb = xgb_00.predict(dtest)\n",
    "\n",
    "\n",
    "# Get class probabilities\n",
    "# Assuming binary classification, get the probability for the positive class (class 1)\n",
    "preds_xgb_margin = xgb_00.predict(dtest, output_margin=True)\n",
    "preds_xgb_prob = 1 / (1 + np.exp(-preds_xgb_margin))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0.0     1.0    2.0    3.0  4.0\n",
      "0.0  2350.0     6.0    2.0    9.0  0.0\n",
      "1.0     5.0  1598.0    0.0    0.0  0.0\n",
      "2.0    10.0     3.0  391.0    2.0  0.0\n",
      "3.0     5.0     0.0    1.0  117.0  0.0\n",
      "4.0     0.0     0.0    0.0    0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9904444444444445\n",
      "Precision total:  0.9784817940829201\n",
      "Recall total:  0.9807944921532954\n",
      "F1 total:  0.9795197389737463\n",
      "BACC total:  0.9807944921532954\n",
      "MCC total:  0.9837320701487882\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    pred_label = preds_xgb\n",
    "    name = 'xgb'\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    metrics = confusion_metrics(name, pred_label, y_test,time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "    globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2312.0    26.0   14.0   15.0  0.0\n",
      "1    13.0  1589.0    1.0    0.0  0.0\n",
      "2    21.0     7.0  375.0    3.0  0.0\n",
      "3    20.0     2.0    1.0  100.0  0.0\n",
      "4     1.0     0.0    0.0    0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9724444444444444\n",
      "Precision total:  0.7523498045895212\n",
      "Recall total:  0.7409367323807244\n",
      "F1 total:  0.7464967662382402\n",
      "BACC total:  0.7409367323807244\n",
      "MCC total:  0.9530241166956729\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_00 = LogisticRegression()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start_lr = start = time.time()\n",
    "    logreg_00.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_lr = preds_logreg =logreg_00.predict(X_test)\n",
    "    preds_lr_prob = logreg_00.predict_proba(X_test)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "    pred_label = preds_logreg\n",
    "    name = 'lr'\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lr\n",
    "    metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    \n",
    "\n",
    "    globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging DT  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2359.0     3.0    2.0    3.0  0.0\n",
      "1     6.0  1597.0    0.0    0.0  0.0\n",
      "2    14.0     3.0  389.0    0.0  0.0\n",
      "3     6.0     0.0    1.0  116.0  0.0\n",
      "4     0.0     0.0    0.0    0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9915555555555555\n",
      "Precision total:  0.990498481065547\n",
      "Recall total:  0.9788189444283896\n",
      "F1 total:  0.9845434330490013\n",
      "BACC total:  0.9788189444283896\n",
      "MCC total:  0.9856103618364205\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "\n",
    "pred_label = y_pred\n",
    "name = 'bag_dt'\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2     3    4\n",
      "0  2294.0    26.0   26.0  21.0  0.0\n",
      "1    24.0  1576.0    3.0   0.0  0.0\n",
      "2    33.0    14.0  359.0   0.0  0.0\n",
      "3    44.0     1.0    0.0  78.0  0.0\n",
      "4     0.0     0.0    0.0   0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9573333333333334\n",
      "Precision total:  0.9291219466148519\n",
      "Recall total:  0.8941397298834047\n",
      "F1 total:  0.9098655073869489\n",
      "BACC total:  0.8941397298834047\n",
      "MCC total:  0.9269937310596706\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_svm'\n",
    "pred_label = y_pred\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test,time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2339.0     9.0    5.0   14.0  0.0\n",
      "1     4.0  1599.0    0.0    0.0  0.0\n",
      "2    13.0     3.0  388.0    2.0  0.0\n",
      "3     8.0     0.0    0.0  114.0  1.0\n",
      "4     0.0     0.0    0.0    0.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9868888888888889\n",
      "Precision total:  0.8692352689865317\n",
      "Recall total:  0.9736339303673001\n",
      "F1 total:  0.9045771058188462\n",
      "BACC total:  0.9736339303673001\n",
      "MCC total:  0.9776943846035531\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "# create MLPClassifier instance\n",
    "mlp_00 = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "base_classifier = mlp_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_mlp'\n",
    "pred_label = y_pred\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bagging KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2326.0    18.0   12.0   11.0  0.0\n",
      "1     1.0  1599.0    2.0    1.0  0.0\n",
      "2    11.0    18.0  377.0    0.0  0.0\n",
      "3    13.0     0.0    0.0  110.0  0.0\n",
      "4     1.0     0.0    0.0    0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9804444444444445\n",
      "Precision total:  0.7665521894542537\n",
      "Recall total:  0.7606127092749453\n",
      "F1 total:  0.7634911269342327\n",
      "BACC total:  0.7606127092749453\n",
      "MCC total:  0.9667499221642365\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "knn_00=KNeighborsClassifier(n_neighbors = 5)\n",
    "start = time.time()\n",
    "base_classifier = knn_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_knn'\n",
    "\n",
    "pred_label = y_pred\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### bag LogRegression\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining baggin Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2311.0    27.0   14.0   15.0  0.0\n",
      "1    16.0  1586.0    1.0    0.0  0.0\n",
      "2    21.0     7.0  374.0    4.0  0.0\n",
      "3    19.0     2.0    1.0  101.0  0.0\n",
      "4     1.0     0.0    0.0    0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9715555555555555\n",
      "Precision total:  0.750875051697339\n",
      "Recall total:  0.7416113444710273\n",
      "F1 total:  0.7461350535549103\n",
      "BACC total:  0.7416113444710273\n",
      "MCC total:  0.9515090692138751\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining baggin Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_00 = LogisticRegression()\n",
    "\n",
    "\n",
    "base_classifier = logreg_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lr'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2    3    4\n",
      "0  2363.0     2.0    2.0  0.0  0.0\n",
      "1    70.0  1533.0    0.0  0.0  0.0\n",
      "2    42.0     8.0  356.0  0.0  0.0\n",
      "3   108.0     3.0    3.0  9.0  0.0\n",
      "4     1.0     0.0    0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9468888888888889\n",
      "Precision total:  0.7784428943605695\n",
      "Recall total:  0.5809319994492763\n",
      "F1 total:  0.5985705489787949\n",
      "BACC total:  0.5809319994492763\n",
      "MCC total:  0.9097403802504108\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "\n",
    "base_classifier = rf\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_rf'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0      1      2     3    4\n",
      "0  2060.0  186.0  116.0   4.0  1.0\n",
      "1    54.0  781.0  768.0   0.0  0.0\n",
      "2   123.0   12.0  271.0   0.0  0.0\n",
      "3    60.0   17.0   10.0  36.0  0.0\n",
      "4     1.0    0.0    0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.6995555555555556\n",
      "Precision total:  0.5626372503314618\n",
      "Recall total:  0.4635364095577231\n",
      "F1 total:  0.45417878409795076\n",
      "BACC total:  0.4635364095577231\n",
      "MCC total:  0.5429096194742403\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "\n",
    "ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "base_classifier = ada\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_ada'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2     3    4\n",
      "0  2358.0     7.0    0.0   2.0  0.0\n",
      "1    17.0  1586.0    0.0   0.0  0.0\n",
      "2    22.0     4.0  380.0   0.0  0.0\n",
      "3    35.0     2.0    0.0  86.0  0.0\n",
      "4     1.0     0.0    0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.98\n",
      "Precision total:  0.7876633010809426\n",
      "Recall total:  0.7241480372450987\n",
      "F1 total:  0.7510435704170609\n",
      "BACC total:  0.7241480372450987\n",
      "MCC total:  0.9658935534784219\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "base_classifier = lgbm\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lgbm'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging Catboost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3254632\ttotal: 17ms\tremaining: 1.68s\n",
      "1:\tlearn: 1.1239883\ttotal: 30.5ms\tremaining: 1.5s\n",
      "2:\tlearn: 0.9952761\ttotal: 42.5ms\tremaining: 1.37s\n",
      "3:\tlearn: 0.8791508\ttotal: 51.3ms\tremaining: 1.23s\n",
      "4:\tlearn: 0.7899724\ttotal: 59.8ms\tremaining: 1.14s\n",
      "5:\tlearn: 0.7086156\ttotal: 66.7ms\tremaining: 1.04s\n",
      "6:\tlearn: 0.6408340\ttotal: 73.8ms\tremaining: 980ms\n",
      "7:\tlearn: 0.5842988\ttotal: 80ms\tremaining: 920ms\n",
      "8:\tlearn: 0.5328991\ttotal: 86.2ms\tremaining: 872ms\n",
      "9:\tlearn: 0.4885743\ttotal: 92.5ms\tremaining: 832ms\n",
      "10:\tlearn: 0.4540532\ttotal: 98.1ms\tremaining: 794ms\n",
      "11:\tlearn: 0.4219235\ttotal: 103ms\tremaining: 758ms\n",
      "12:\tlearn: 0.3913296\ttotal: 109ms\tremaining: 732ms\n",
      "13:\tlearn: 0.3638104\ttotal: 115ms\tremaining: 706ms\n",
      "14:\tlearn: 0.3415363\ttotal: 121ms\tremaining: 683ms\n",
      "15:\tlearn: 0.3193731\ttotal: 127ms\tremaining: 665ms\n",
      "16:\tlearn: 0.2991671\ttotal: 132ms\tremaining: 644ms\n",
      "17:\tlearn: 0.2816184\ttotal: 137ms\tremaining: 625ms\n",
      "18:\tlearn: 0.2651195\ttotal: 142ms\tremaining: 606ms\n",
      "19:\tlearn: 0.2513871\ttotal: 148ms\tremaining: 590ms\n",
      "20:\tlearn: 0.2373901\ttotal: 153ms\tremaining: 574ms\n",
      "21:\tlearn: 0.2255234\ttotal: 158ms\tremaining: 559ms\n",
      "22:\tlearn: 0.2148819\ttotal: 163ms\tremaining: 546ms\n",
      "23:\tlearn: 0.2040597\ttotal: 168ms\tremaining: 533ms\n",
      "24:\tlearn: 0.1947581\ttotal: 173ms\tremaining: 520ms\n",
      "25:\tlearn: 0.1877852\ttotal: 178ms\tremaining: 508ms\n",
      "26:\tlearn: 0.1779531\ttotal: 184ms\tremaining: 496ms\n",
      "27:\tlearn: 0.1702549\ttotal: 189ms\tremaining: 485ms\n",
      "28:\tlearn: 0.1635638\ttotal: 194ms\tremaining: 475ms\n",
      "29:\tlearn: 0.1584688\ttotal: 199ms\tremaining: 465ms\n",
      "30:\tlearn: 0.1534237\ttotal: 204ms\tremaining: 454ms\n",
      "31:\tlearn: 0.1484695\ttotal: 209ms\tremaining: 445ms\n",
      "32:\tlearn: 0.1433105\ttotal: 214ms\tremaining: 435ms\n",
      "33:\tlearn: 0.1387870\ttotal: 220ms\tremaining: 426ms\n",
      "34:\tlearn: 0.1343184\ttotal: 225ms\tremaining: 418ms\n",
      "35:\tlearn: 0.1304676\ttotal: 230ms\tremaining: 409ms\n",
      "36:\tlearn: 0.1262423\ttotal: 235ms\tremaining: 400ms\n",
      "37:\tlearn: 0.1233790\ttotal: 240ms\tremaining: 391ms\n",
      "38:\tlearn: 0.1196038\ttotal: 245ms\tremaining: 383ms\n",
      "39:\tlearn: 0.1173554\ttotal: 249ms\tremaining: 374ms\n",
      "40:\tlearn: 0.1151772\ttotal: 254ms\tremaining: 366ms\n",
      "41:\tlearn: 0.1125143\ttotal: 259ms\tremaining: 358ms\n",
      "42:\tlearn: 0.1096505\ttotal: 264ms\tremaining: 351ms\n",
      "43:\tlearn: 0.1073783\ttotal: 269ms\tremaining: 342ms\n",
      "44:\tlearn: 0.1043932\ttotal: 274ms\tremaining: 335ms\n",
      "45:\tlearn: 0.1012212\ttotal: 279ms\tremaining: 328ms\n",
      "46:\tlearn: 0.0987468\ttotal: 284ms\tremaining: 321ms\n",
      "47:\tlearn: 0.0970959\ttotal: 289ms\tremaining: 313ms\n",
      "48:\tlearn: 0.0948651\ttotal: 295ms\tremaining: 307ms\n",
      "49:\tlearn: 0.0926852\ttotal: 300ms\tremaining: 300ms\n",
      "50:\tlearn: 0.0909648\ttotal: 304ms\tremaining: 293ms\n",
      "51:\tlearn: 0.0890872\ttotal: 309ms\tremaining: 286ms\n",
      "52:\tlearn: 0.0873456\ttotal: 314ms\tremaining: 279ms\n",
      "53:\tlearn: 0.0858786\ttotal: 319ms\tremaining: 272ms\n",
      "54:\tlearn: 0.0848654\ttotal: 324ms\tremaining: 265ms\n",
      "55:\tlearn: 0.0837738\ttotal: 329ms\tremaining: 258ms\n",
      "56:\tlearn: 0.0825749\ttotal: 333ms\tremaining: 251ms\n",
      "57:\tlearn: 0.0816011\ttotal: 338ms\tremaining: 245ms\n",
      "58:\tlearn: 0.0794240\ttotal: 343ms\tremaining: 238ms\n",
      "59:\tlearn: 0.0782945\ttotal: 348ms\tremaining: 232ms\n",
      "60:\tlearn: 0.0774684\ttotal: 353ms\tremaining: 225ms\n",
      "61:\tlearn: 0.0763655\ttotal: 358ms\tremaining: 219ms\n",
      "62:\tlearn: 0.0751815\ttotal: 362ms\tremaining: 213ms\n",
      "63:\tlearn: 0.0737115\ttotal: 368ms\tremaining: 207ms\n",
      "64:\tlearn: 0.0724516\ttotal: 373ms\tremaining: 201ms\n",
      "65:\tlearn: 0.0720632\ttotal: 377ms\tremaining: 194ms\n",
      "66:\tlearn: 0.0707253\ttotal: 382ms\tremaining: 188ms\n",
      "67:\tlearn: 0.0699700\ttotal: 387ms\tremaining: 182ms\n",
      "68:\tlearn: 0.0688898\ttotal: 392ms\tremaining: 176ms\n",
      "69:\tlearn: 0.0685348\ttotal: 397ms\tremaining: 170ms\n",
      "70:\tlearn: 0.0676430\ttotal: 401ms\tremaining: 164ms\n",
      "71:\tlearn: 0.0666731\ttotal: 407ms\tremaining: 158ms\n",
      "72:\tlearn: 0.0665097\ttotal: 411ms\tremaining: 152ms\n",
      "73:\tlearn: 0.0660161\ttotal: 416ms\tremaining: 146ms\n",
      "74:\tlearn: 0.0655307\ttotal: 421ms\tremaining: 140ms\n",
      "75:\tlearn: 0.0652094\ttotal: 426ms\tremaining: 134ms\n",
      "76:\tlearn: 0.0646516\ttotal: 430ms\tremaining: 129ms\n",
      "77:\tlearn: 0.0638265\ttotal: 435ms\tremaining: 123ms\n",
      "78:\tlearn: 0.0636175\ttotal: 440ms\tremaining: 117ms\n",
      "79:\tlearn: 0.0631495\ttotal: 444ms\tremaining: 111ms\n",
      "80:\tlearn: 0.0624576\ttotal: 450ms\tremaining: 106ms\n",
      "81:\tlearn: 0.0617264\ttotal: 455ms\tremaining: 99.9ms\n",
      "82:\tlearn: 0.0607709\ttotal: 460ms\tremaining: 94.2ms\n",
      "83:\tlearn: 0.0606077\ttotal: 464ms\tremaining: 88.5ms\n",
      "84:\tlearn: 0.0602729\ttotal: 469ms\tremaining: 82.7ms\n",
      "85:\tlearn: 0.0596256\ttotal: 474ms\tremaining: 77.1ms\n",
      "86:\tlearn: 0.0590465\ttotal: 478ms\tremaining: 71.4ms\n",
      "87:\tlearn: 0.0589143\ttotal: 483ms\tremaining: 65.8ms\n",
      "88:\tlearn: 0.0580788\ttotal: 488ms\tremaining: 60.3ms\n",
      "89:\tlearn: 0.0574857\ttotal: 493ms\tremaining: 54.7ms\n",
      "90:\tlearn: 0.0570724\ttotal: 497ms\tremaining: 49.2ms\n",
      "91:\tlearn: 0.0564694\ttotal: 502ms\tremaining: 43.7ms\n",
      "92:\tlearn: 0.0564374\ttotal: 507ms\tremaining: 38.1ms\n",
      "93:\tlearn: 0.0554287\ttotal: 512ms\tremaining: 32.7ms\n",
      "94:\tlearn: 0.0546869\ttotal: 516ms\tremaining: 27.2ms\n",
      "95:\tlearn: 0.0541586\ttotal: 522ms\tremaining: 21.7ms\n",
      "96:\tlearn: 0.0537711\ttotal: 526ms\tremaining: 16.3ms\n",
      "97:\tlearn: 0.0533353\ttotal: 531ms\tremaining: 10.8ms\n",
      "98:\tlearn: 0.0529488\ttotal: 534ms\tremaining: 5.4ms\n",
      "99:\tlearn: 0.0522540\ttotal: 539ms\tremaining: 0us\n",
      "0:\tlearn: 1.3451290\ttotal: 16.8ms\tremaining: 1.67s\n",
      "1:\tlearn: 1.1318283\ttotal: 29ms\tremaining: 1.42s\n",
      "2:\tlearn: 0.9994674\ttotal: 41.4ms\tremaining: 1.34s\n",
      "3:\tlearn: 0.8771748\ttotal: 51.8ms\tremaining: 1.24s\n",
      "4:\tlearn: 0.7878485\ttotal: 60.9ms\tremaining: 1.16s\n",
      "5:\tlearn: 0.7109968\ttotal: 68.2ms\tremaining: 1.07s\n",
      "6:\tlearn: 0.6456499\ttotal: 75.4ms\tremaining: 1s\n",
      "7:\tlearn: 0.5852876\ttotal: 82ms\tremaining: 943ms\n",
      "8:\tlearn: 0.5335978\ttotal: 88.5ms\tremaining: 894ms\n",
      "9:\tlearn: 0.4895742\ttotal: 94.9ms\tremaining: 854ms\n",
      "10:\tlearn: 0.4519400\ttotal: 100ms\tremaining: 812ms\n",
      "11:\tlearn: 0.4221545\ttotal: 106ms\tremaining: 775ms\n",
      "12:\tlearn: 0.3902831\ttotal: 112ms\tremaining: 750ms\n",
      "13:\tlearn: 0.3646188\ttotal: 117ms\tremaining: 720ms\n",
      "14:\tlearn: 0.3436373\ttotal: 122ms\tremaining: 693ms\n",
      "15:\tlearn: 0.3215341\ttotal: 128ms\tremaining: 670ms\n",
      "16:\tlearn: 0.3042313\ttotal: 133ms\tremaining: 649ms\n",
      "17:\tlearn: 0.2864275\ttotal: 138ms\tremaining: 630ms\n",
      "18:\tlearn: 0.2702654\ttotal: 144ms\tremaining: 613ms\n",
      "19:\tlearn: 0.2545471\ttotal: 149ms\tremaining: 598ms\n",
      "20:\tlearn: 0.2403054\ttotal: 155ms\tremaining: 582ms\n",
      "21:\tlearn: 0.2287447\ttotal: 160ms\tremaining: 568ms\n",
      "22:\tlearn: 0.2167714\ttotal: 166ms\tremaining: 555ms\n",
      "23:\tlearn: 0.2079552\ttotal: 171ms\tremaining: 541ms\n",
      "24:\tlearn: 0.1979186\ttotal: 177ms\tremaining: 530ms\n",
      "25:\tlearn: 0.1915031\ttotal: 181ms\tremaining: 516ms\n",
      "26:\tlearn: 0.1813474\ttotal: 187ms\tremaining: 505ms\n",
      "27:\tlearn: 0.1739380\ttotal: 192ms\tremaining: 494ms\n",
      "28:\tlearn: 0.1669509\ttotal: 197ms\tremaining: 483ms\n",
      "29:\tlearn: 0.1605887\ttotal: 203ms\tremaining: 473ms\n",
      "30:\tlearn: 0.1558918\ttotal: 208ms\tremaining: 462ms\n",
      "31:\tlearn: 0.1504661\ttotal: 213ms\tremaining: 452ms\n",
      "32:\tlearn: 0.1461860\ttotal: 218ms\tremaining: 442ms\n",
      "33:\tlearn: 0.1419992\ttotal: 223ms\tremaining: 433ms\n",
      "34:\tlearn: 0.1383113\ttotal: 228ms\tremaining: 423ms\n",
      "35:\tlearn: 0.1342664\ttotal: 233ms\tremaining: 415ms\n",
      "36:\tlearn: 0.1298465\ttotal: 238ms\tremaining: 405ms\n",
      "37:\tlearn: 0.1255048\ttotal: 243ms\tremaining: 397ms\n",
      "38:\tlearn: 0.1218505\ttotal: 249ms\tremaining: 389ms\n",
      "39:\tlearn: 0.1186637\ttotal: 254ms\tremaining: 380ms\n",
      "40:\tlearn: 0.1153298\ttotal: 259ms\tremaining: 372ms\n",
      "41:\tlearn: 0.1121926\ttotal: 263ms\tremaining: 364ms\n",
      "42:\tlearn: 0.1098637\ttotal: 268ms\tremaining: 356ms\n",
      "43:\tlearn: 0.1070670\ttotal: 274ms\tremaining: 348ms\n",
      "44:\tlearn: 0.1048035\ttotal: 278ms\tremaining: 340ms\n",
      "45:\tlearn: 0.1024589\ttotal: 283ms\tremaining: 333ms\n",
      "46:\tlearn: 0.1008306\ttotal: 288ms\tremaining: 325ms\n",
      "47:\tlearn: 0.0985617\ttotal: 293ms\tremaining: 318ms\n",
      "48:\tlearn: 0.0959093\ttotal: 298ms\tremaining: 311ms\n",
      "49:\tlearn: 0.0944142\ttotal: 303ms\tremaining: 303ms\n",
      "50:\tlearn: 0.0933138\ttotal: 309ms\tremaining: 297ms\n",
      "51:\tlearn: 0.0913039\ttotal: 314ms\tremaining: 290ms\n",
      "52:\tlearn: 0.0891398\ttotal: 319ms\tremaining: 283ms\n",
      "53:\tlearn: 0.0878706\ttotal: 324ms\tremaining: 276ms\n",
      "54:\tlearn: 0.0861490\ttotal: 329ms\tremaining: 269ms\n",
      "55:\tlearn: 0.0848461\ttotal: 334ms\tremaining: 262ms\n",
      "56:\tlearn: 0.0833480\ttotal: 339ms\tremaining: 256ms\n",
      "57:\tlearn: 0.0816294\ttotal: 344ms\tremaining: 249ms\n",
      "58:\tlearn: 0.0808644\ttotal: 349ms\tremaining: 242ms\n",
      "59:\tlearn: 0.0801934\ttotal: 353ms\tremaining: 235ms\n",
      "60:\tlearn: 0.0788657\ttotal: 358ms\tremaining: 229ms\n",
      "61:\tlearn: 0.0782174\ttotal: 363ms\tremaining: 222ms\n",
      "62:\tlearn: 0.0776144\ttotal: 367ms\tremaining: 216ms\n",
      "63:\tlearn: 0.0765312\ttotal: 372ms\tremaining: 209ms\n",
      "64:\tlearn: 0.0754140\ttotal: 377ms\tremaining: 203ms\n",
      "65:\tlearn: 0.0751091\ttotal: 382ms\tremaining: 197ms\n",
      "66:\tlearn: 0.0740858\ttotal: 387ms\tremaining: 191ms\n",
      "67:\tlearn: 0.0733140\ttotal: 392ms\tremaining: 184ms\n",
      "68:\tlearn: 0.0725817\ttotal: 396ms\tremaining: 178ms\n",
      "69:\tlearn: 0.0723108\ttotal: 401ms\tremaining: 172ms\n",
      "70:\tlearn: 0.0713267\ttotal: 406ms\tremaining: 166ms\n",
      "71:\tlearn: 0.0707686\ttotal: 411ms\tremaining: 160ms\n",
      "72:\tlearn: 0.0704712\ttotal: 416ms\tremaining: 154ms\n",
      "73:\tlearn: 0.0698241\ttotal: 421ms\tremaining: 148ms\n",
      "74:\tlearn: 0.0693098\ttotal: 426ms\tremaining: 142ms\n",
      "75:\tlearn: 0.0688576\ttotal: 430ms\tremaining: 136ms\n",
      "76:\tlearn: 0.0682266\ttotal: 435ms\tremaining: 130ms\n",
      "77:\tlearn: 0.0676496\ttotal: 440ms\tremaining: 124ms\n",
      "78:\tlearn: 0.0664709\ttotal: 445ms\tremaining: 118ms\n",
      "79:\tlearn: 0.0658078\ttotal: 450ms\tremaining: 112ms\n",
      "80:\tlearn: 0.0646821\ttotal: 455ms\tremaining: 107ms\n",
      "81:\tlearn: 0.0640731\ttotal: 460ms\tremaining: 101ms\n",
      "82:\tlearn: 0.0635386\ttotal: 465ms\tremaining: 95.2ms\n",
      "83:\tlearn: 0.0627075\ttotal: 470ms\tremaining: 89.5ms\n",
      "84:\tlearn: 0.0622434\ttotal: 475ms\tremaining: 83.8ms\n",
      "85:\tlearn: 0.0613442\ttotal: 480ms\tremaining: 78.1ms\n",
      "86:\tlearn: 0.0610695\ttotal: 485ms\tremaining: 72.4ms\n",
      "87:\tlearn: 0.0606025\ttotal: 490ms\tremaining: 66.8ms\n",
      "88:\tlearn: 0.0600671\ttotal: 495ms\tremaining: 61.1ms\n",
      "89:\tlearn: 0.0590570\ttotal: 499ms\tremaining: 55.5ms\n",
      "90:\tlearn: 0.0584126\ttotal: 504ms\tremaining: 49.9ms\n",
      "91:\tlearn: 0.0578979\ttotal: 510ms\tremaining: 44.3ms\n",
      "92:\tlearn: 0.0576009\ttotal: 514ms\tremaining: 38.7ms\n",
      "93:\tlearn: 0.0570177\ttotal: 519ms\tremaining: 33.1ms\n",
      "94:\tlearn: 0.0568185\ttotal: 524ms\tremaining: 27.6ms\n",
      "95:\tlearn: 0.0559732\ttotal: 529ms\tremaining: 22ms\n",
      "96:\tlearn: 0.0554683\ttotal: 534ms\tremaining: 16.5ms\n",
      "97:\tlearn: 0.0549621\ttotal: 539ms\tremaining: 11ms\n",
      "98:\tlearn: 0.0545063\ttotal: 544ms\tremaining: 5.49ms\n",
      "99:\tlearn: 0.0541044\ttotal: 548ms\tremaining: 0us\n",
      "0:\tlearn: 1.3219477\ttotal: 18.3ms\tremaining: 1.81s\n",
      "1:\tlearn: 1.1189592\ttotal: 31.8ms\tremaining: 1.56s\n",
      "2:\tlearn: 0.9880495\ttotal: 45.5ms\tremaining: 1.47s\n",
      "3:\tlearn: 0.8660446\ttotal: 55.5ms\tremaining: 1.33s\n",
      "4:\tlearn: 0.7775621\ttotal: 64.1ms\tremaining: 1.22s\n",
      "5:\tlearn: 0.7006103\ttotal: 71.6ms\tremaining: 1.12s\n",
      "6:\tlearn: 0.6332391\ttotal: 78.9ms\tremaining: 1.05s\n",
      "7:\tlearn: 0.5769338\ttotal: 86.3ms\tremaining: 992ms\n",
      "8:\tlearn: 0.5258126\ttotal: 93.1ms\tremaining: 941ms\n",
      "9:\tlearn: 0.4834912\ttotal: 99.2ms\tremaining: 892ms\n",
      "10:\tlearn: 0.4485583\ttotal: 105ms\tremaining: 852ms\n",
      "11:\tlearn: 0.4173707\ttotal: 111ms\tremaining: 811ms\n",
      "12:\tlearn: 0.3875746\ttotal: 116ms\tremaining: 779ms\n",
      "13:\tlearn: 0.3625765\ttotal: 122ms\tremaining: 747ms\n",
      "14:\tlearn: 0.3407727\ttotal: 127ms\tremaining: 720ms\n",
      "15:\tlearn: 0.3174520\ttotal: 133ms\tremaining: 698ms\n",
      "16:\tlearn: 0.2987035\ttotal: 138ms\tremaining: 673ms\n",
      "17:\tlearn: 0.2813066\ttotal: 143ms\tremaining: 652ms\n",
      "18:\tlearn: 0.2641880\ttotal: 149ms\tremaining: 635ms\n",
      "19:\tlearn: 0.2496230\ttotal: 154ms\tremaining: 618ms\n",
      "20:\tlearn: 0.2364643\ttotal: 160ms\tremaining: 601ms\n",
      "21:\tlearn: 0.2247195\ttotal: 165ms\tremaining: 585ms\n",
      "22:\tlearn: 0.2135640\ttotal: 170ms\tremaining: 570ms\n",
      "23:\tlearn: 0.2040420\ttotal: 175ms\tremaining: 556ms\n",
      "24:\tlearn: 0.1944537\ttotal: 181ms\tremaining: 543ms\n",
      "25:\tlearn: 0.1855252\ttotal: 186ms\tremaining: 530ms\n",
      "26:\tlearn: 0.1765396\ttotal: 191ms\tremaining: 517ms\n",
      "27:\tlearn: 0.1701463\ttotal: 196ms\tremaining: 504ms\n",
      "28:\tlearn: 0.1637880\ttotal: 201ms\tremaining: 493ms\n",
      "29:\tlearn: 0.1576986\ttotal: 207ms\tremaining: 483ms\n",
      "30:\tlearn: 0.1518419\ttotal: 212ms\tremaining: 472ms\n",
      "31:\tlearn: 0.1470673\ttotal: 218ms\tremaining: 462ms\n",
      "32:\tlearn: 0.1425581\ttotal: 222ms\tremaining: 451ms\n",
      "33:\tlearn: 0.1386222\ttotal: 227ms\tremaining: 440ms\n",
      "34:\tlearn: 0.1346127\ttotal: 232ms\tremaining: 430ms\n",
      "35:\tlearn: 0.1307389\ttotal: 237ms\tremaining: 421ms\n",
      "36:\tlearn: 0.1275127\ttotal: 242ms\tremaining: 411ms\n",
      "37:\tlearn: 0.1235475\ttotal: 247ms\tremaining: 402ms\n",
      "38:\tlearn: 0.1199690\ttotal: 253ms\tremaining: 395ms\n",
      "39:\tlearn: 0.1157860\ttotal: 258ms\tremaining: 387ms\n",
      "40:\tlearn: 0.1127813\ttotal: 263ms\tremaining: 378ms\n",
      "41:\tlearn: 0.1094700\ttotal: 268ms\tremaining: 370ms\n",
      "42:\tlearn: 0.1073641\ttotal: 273ms\tremaining: 362ms\n",
      "43:\tlearn: 0.1043462\ttotal: 278ms\tremaining: 354ms\n",
      "44:\tlearn: 0.1021599\ttotal: 283ms\tremaining: 346ms\n",
      "45:\tlearn: 0.0994165\ttotal: 288ms\tremaining: 338ms\n",
      "46:\tlearn: 0.0979331\ttotal: 293ms\tremaining: 330ms\n",
      "47:\tlearn: 0.0951134\ttotal: 298ms\tremaining: 323ms\n",
      "48:\tlearn: 0.0932211\ttotal: 303ms\tremaining: 315ms\n",
      "49:\tlearn: 0.0909361\ttotal: 308ms\tremaining: 308ms\n",
      "50:\tlearn: 0.0897276\ttotal: 312ms\tremaining: 300ms\n",
      "51:\tlearn: 0.0876861\ttotal: 317ms\tremaining: 293ms\n",
      "52:\tlearn: 0.0855101\ttotal: 323ms\tremaining: 286ms\n",
      "53:\tlearn: 0.0840909\ttotal: 328ms\tremaining: 279ms\n",
      "54:\tlearn: 0.0826816\ttotal: 333ms\tremaining: 272ms\n",
      "55:\tlearn: 0.0808977\ttotal: 338ms\tremaining: 266ms\n",
      "56:\tlearn: 0.0797270\ttotal: 343ms\tremaining: 259ms\n",
      "57:\tlearn: 0.0789868\ttotal: 348ms\tremaining: 252ms\n",
      "58:\tlearn: 0.0776556\ttotal: 352ms\tremaining: 245ms\n",
      "59:\tlearn: 0.0767234\ttotal: 357ms\tremaining: 238ms\n",
      "60:\tlearn: 0.0759878\ttotal: 362ms\tremaining: 231ms\n",
      "61:\tlearn: 0.0751654\ttotal: 367ms\tremaining: 225ms\n",
      "62:\tlearn: 0.0731499\ttotal: 372ms\tremaining: 218ms\n",
      "63:\tlearn: 0.0723585\ttotal: 377ms\tremaining: 212ms\n",
      "64:\tlearn: 0.0712235\ttotal: 382ms\tremaining: 206ms\n",
      "65:\tlearn: 0.0697402\ttotal: 387ms\tremaining: 199ms\n",
      "66:\tlearn: 0.0682772\ttotal: 392ms\tremaining: 193ms\n",
      "67:\tlearn: 0.0667904\ttotal: 397ms\tremaining: 187ms\n",
      "68:\tlearn: 0.0657256\ttotal: 402ms\tremaining: 180ms\n",
      "69:\tlearn: 0.0648191\ttotal: 407ms\tremaining: 174ms\n",
      "70:\tlearn: 0.0643147\ttotal: 412ms\tremaining: 168ms\n",
      "71:\tlearn: 0.0629845\ttotal: 417ms\tremaining: 162ms\n",
      "72:\tlearn: 0.0621770\ttotal: 422ms\tremaining: 156ms\n",
      "73:\tlearn: 0.0615641\ttotal: 427ms\tremaining: 150ms\n",
      "74:\tlearn: 0.0610538\ttotal: 431ms\tremaining: 144ms\n",
      "75:\tlearn: 0.0606014\ttotal: 436ms\tremaining: 138ms\n",
      "76:\tlearn: 0.0599354\ttotal: 441ms\tremaining: 132ms\n",
      "77:\tlearn: 0.0589020\ttotal: 445ms\tremaining: 126ms\n",
      "78:\tlearn: 0.0580653\ttotal: 450ms\tremaining: 120ms\n",
      "79:\tlearn: 0.0571728\ttotal: 455ms\tremaining: 114ms\n",
      "80:\tlearn: 0.0560933\ttotal: 460ms\tremaining: 108ms\n",
      "81:\tlearn: 0.0556486\ttotal: 465ms\tremaining: 102ms\n",
      "82:\tlearn: 0.0551067\ttotal: 470ms\tremaining: 96.3ms\n",
      "83:\tlearn: 0.0544981\ttotal: 476ms\tremaining: 90.6ms\n",
      "84:\tlearn: 0.0539274\ttotal: 481ms\tremaining: 84.9ms\n",
      "85:\tlearn: 0.0533230\ttotal: 486ms\tremaining: 79.1ms\n",
      "86:\tlearn: 0.0529352\ttotal: 491ms\tremaining: 73.3ms\n",
      "87:\tlearn: 0.0527859\ttotal: 495ms\tremaining: 67.6ms\n",
      "88:\tlearn: 0.0525516\ttotal: 500ms\tremaining: 61.8ms\n",
      "89:\tlearn: 0.0517680\ttotal: 505ms\tremaining: 56.1ms\n",
      "90:\tlearn: 0.0514797\ttotal: 510ms\tremaining: 50.5ms\n",
      "91:\tlearn: 0.0512217\ttotal: 515ms\tremaining: 44.8ms\n",
      "92:\tlearn: 0.0510825\ttotal: 520ms\tremaining: 39.1ms\n",
      "93:\tlearn: 0.0506151\ttotal: 525ms\tremaining: 33.5ms\n",
      "94:\tlearn: 0.0498852\ttotal: 530ms\tremaining: 27.9ms\n",
      "95:\tlearn: 0.0494332\ttotal: 535ms\tremaining: 22.3ms\n",
      "96:\tlearn: 0.0488757\ttotal: 540ms\tremaining: 16.7ms\n",
      "97:\tlearn: 0.0482725\ttotal: 545ms\tremaining: 11.1ms\n",
      "98:\tlearn: 0.0480823\ttotal: 550ms\tremaining: 5.55ms\n",
      "99:\tlearn: 0.0479209\ttotal: 554ms\tremaining: 0us\n",
      "0:\tlearn: 1.3222713\ttotal: 16.3ms\tremaining: 1.61s\n",
      "1:\tlearn: 1.1162468\ttotal: 28.7ms\tremaining: 1.41s\n",
      "2:\tlearn: 0.9857067\ttotal: 41.4ms\tremaining: 1.34s\n",
      "3:\tlearn: 0.8661073\ttotal: 52ms\tremaining: 1.25s\n",
      "4:\tlearn: 0.7784680\ttotal: 60.4ms\tremaining: 1.15s\n",
      "5:\tlearn: 0.7011105\ttotal: 68.7ms\tremaining: 1.07s\n",
      "6:\tlearn: 0.6347854\ttotal: 76.2ms\tremaining: 1.01s\n",
      "7:\tlearn: 0.5774074\ttotal: 82.9ms\tremaining: 953ms\n",
      "8:\tlearn: 0.5247758\ttotal: 89.7ms\tremaining: 907ms\n",
      "9:\tlearn: 0.4846484\ttotal: 96ms\tremaining: 864ms\n",
      "10:\tlearn: 0.4493261\ttotal: 102ms\tremaining: 827ms\n",
      "11:\tlearn: 0.4160730\ttotal: 108ms\tremaining: 792ms\n",
      "12:\tlearn: 0.3871360\ttotal: 114ms\tremaining: 763ms\n",
      "13:\tlearn: 0.3612848\ttotal: 120ms\tremaining: 736ms\n",
      "14:\tlearn: 0.3388262\ttotal: 125ms\tremaining: 709ms\n",
      "15:\tlearn: 0.3165338\ttotal: 131ms\tremaining: 688ms\n",
      "16:\tlearn: 0.2978222\ttotal: 136ms\tremaining: 665ms\n",
      "17:\tlearn: 0.2819037\ttotal: 141ms\tremaining: 644ms\n",
      "18:\tlearn: 0.2677186\ttotal: 147ms\tremaining: 625ms\n",
      "19:\tlearn: 0.2523725\ttotal: 152ms\tremaining: 608ms\n",
      "20:\tlearn: 0.2373228\ttotal: 157ms\tremaining: 592ms\n",
      "21:\tlearn: 0.2264323\ttotal: 163ms\tremaining: 577ms\n",
      "22:\tlearn: 0.2159107\ttotal: 168ms\tremaining: 563ms\n",
      "23:\tlearn: 0.2059886\ttotal: 174ms\tremaining: 550ms\n",
      "24:\tlearn: 0.1976281\ttotal: 179ms\tremaining: 537ms\n",
      "25:\tlearn: 0.1902677\ttotal: 184ms\tremaining: 523ms\n",
      "26:\tlearn: 0.1835933\ttotal: 189ms\tremaining: 510ms\n",
      "27:\tlearn: 0.1764059\ttotal: 194ms\tremaining: 499ms\n",
      "28:\tlearn: 0.1696787\ttotal: 199ms\tremaining: 487ms\n",
      "29:\tlearn: 0.1637251\ttotal: 204ms\tremaining: 476ms\n",
      "30:\tlearn: 0.1591040\ttotal: 209ms\tremaining: 465ms\n",
      "31:\tlearn: 0.1537775\ttotal: 214ms\tremaining: 455ms\n",
      "32:\tlearn: 0.1494108\ttotal: 220ms\tremaining: 446ms\n",
      "33:\tlearn: 0.1454176\ttotal: 224ms\tremaining: 436ms\n",
      "34:\tlearn: 0.1418254\ttotal: 229ms\tremaining: 426ms\n",
      "35:\tlearn: 0.1375782\ttotal: 235ms\tremaining: 417ms\n",
      "36:\tlearn: 0.1344451\ttotal: 239ms\tremaining: 408ms\n",
      "37:\tlearn: 0.1306674\ttotal: 245ms\tremaining: 399ms\n",
      "38:\tlearn: 0.1266646\ttotal: 250ms\tremaining: 391ms\n",
      "39:\tlearn: 0.1235778\ttotal: 255ms\tremaining: 382ms\n",
      "40:\tlearn: 0.1209520\ttotal: 260ms\tremaining: 374ms\n",
      "41:\tlearn: 0.1180269\ttotal: 265ms\tremaining: 366ms\n",
      "42:\tlearn: 0.1149708\ttotal: 270ms\tremaining: 358ms\n",
      "43:\tlearn: 0.1127041\ttotal: 275ms\tremaining: 350ms\n",
      "44:\tlearn: 0.1086379\ttotal: 280ms\tremaining: 342ms\n",
      "45:\tlearn: 0.1067489\ttotal: 284ms\tremaining: 334ms\n",
      "46:\tlearn: 0.1034617\ttotal: 290ms\tremaining: 327ms\n",
      "47:\tlearn: 0.1013447\ttotal: 294ms\tremaining: 319ms\n",
      "48:\tlearn: 0.0979173\ttotal: 300ms\tremaining: 312ms\n",
      "49:\tlearn: 0.0954334\ttotal: 305ms\tremaining: 305ms\n",
      "50:\tlearn: 0.0939063\ttotal: 310ms\tremaining: 298ms\n",
      "51:\tlearn: 0.0922772\ttotal: 315ms\tremaining: 291ms\n",
      "52:\tlearn: 0.0905233\ttotal: 320ms\tremaining: 284ms\n",
      "53:\tlearn: 0.0894119\ttotal: 325ms\tremaining: 277ms\n",
      "54:\tlearn: 0.0872897\ttotal: 330ms\tremaining: 270ms\n",
      "55:\tlearn: 0.0864066\ttotal: 335ms\tremaining: 263ms\n",
      "56:\tlearn: 0.0851928\ttotal: 340ms\tremaining: 256ms\n",
      "57:\tlearn: 0.0844815\ttotal: 344ms\tremaining: 249ms\n",
      "58:\tlearn: 0.0827148\ttotal: 350ms\tremaining: 243ms\n",
      "59:\tlearn: 0.0814658\ttotal: 355ms\tremaining: 236ms\n",
      "60:\tlearn: 0.0803694\ttotal: 359ms\tremaining: 230ms\n",
      "61:\tlearn: 0.0796599\ttotal: 364ms\tremaining: 223ms\n",
      "62:\tlearn: 0.0776078\ttotal: 370ms\tremaining: 217ms\n",
      "63:\tlearn: 0.0766308\ttotal: 375ms\tremaining: 211ms\n",
      "64:\tlearn: 0.0760588\ttotal: 379ms\tremaining: 204ms\n",
      "65:\tlearn: 0.0751822\ttotal: 384ms\tremaining: 198ms\n",
      "66:\tlearn: 0.0739983\ttotal: 389ms\tremaining: 192ms\n",
      "67:\tlearn: 0.0729984\ttotal: 394ms\tremaining: 185ms\n",
      "68:\tlearn: 0.0727176\ttotal: 398ms\tremaining: 179ms\n",
      "69:\tlearn: 0.0722236\ttotal: 403ms\tremaining: 173ms\n",
      "70:\tlearn: 0.0716099\ttotal: 407ms\tremaining: 166ms\n",
      "71:\tlearn: 0.0705464\ttotal: 413ms\tremaining: 161ms\n",
      "72:\tlearn: 0.0701827\ttotal: 417ms\tremaining: 154ms\n",
      "73:\tlearn: 0.0694015\ttotal: 422ms\tremaining: 148ms\n",
      "74:\tlearn: 0.0682935\ttotal: 427ms\tremaining: 142ms\n",
      "75:\tlearn: 0.0676099\ttotal: 432ms\tremaining: 136ms\n",
      "76:\tlearn: 0.0670118\ttotal: 436ms\tremaining: 130ms\n",
      "77:\tlearn: 0.0663523\ttotal: 441ms\tremaining: 124ms\n",
      "78:\tlearn: 0.0657972\ttotal: 446ms\tremaining: 119ms\n",
      "79:\tlearn: 0.0652651\ttotal: 451ms\tremaining: 113ms\n",
      "80:\tlearn: 0.0647637\ttotal: 456ms\tremaining: 107ms\n",
      "81:\tlearn: 0.0640114\ttotal: 461ms\tremaining: 101ms\n",
      "82:\tlearn: 0.0633414\ttotal: 465ms\tremaining: 95.3ms\n",
      "83:\tlearn: 0.0627541\ttotal: 470ms\tremaining: 89.5ms\n",
      "84:\tlearn: 0.0622890\ttotal: 475ms\tremaining: 83.8ms\n",
      "85:\tlearn: 0.0618213\ttotal: 479ms\tremaining: 78ms\n",
      "86:\tlearn: 0.0612176\ttotal: 484ms\tremaining: 72.3ms\n",
      "87:\tlearn: 0.0601165\ttotal: 489ms\tremaining: 66.7ms\n",
      "88:\tlearn: 0.0592072\ttotal: 494ms\tremaining: 61.1ms\n",
      "89:\tlearn: 0.0585272\ttotal: 499ms\tremaining: 55.4ms\n",
      "90:\tlearn: 0.0579960\ttotal: 504ms\tremaining: 49.8ms\n",
      "91:\tlearn: 0.0573634\ttotal: 509ms\tremaining: 44.2ms\n",
      "92:\tlearn: 0.0566740\ttotal: 514ms\tremaining: 38.7ms\n",
      "93:\tlearn: 0.0561750\ttotal: 519ms\tremaining: 33.1ms\n",
      "94:\tlearn: 0.0558063\ttotal: 523ms\tremaining: 27.5ms\n",
      "95:\tlearn: 0.0552987\ttotal: 528ms\tremaining: 22ms\n",
      "96:\tlearn: 0.0545548\ttotal: 533ms\tremaining: 16.5ms\n",
      "97:\tlearn: 0.0541321\ttotal: 538ms\tremaining: 11ms\n",
      "98:\tlearn: 0.0537902\ttotal: 543ms\tremaining: 5.48ms\n",
      "99:\tlearn: 0.0533397\ttotal: 547ms\tremaining: 0us\n",
      "0:\tlearn: 1.3303075\ttotal: 16.7ms\tremaining: 1.66s\n",
      "1:\tlearn: 1.1225600\ttotal: 28.3ms\tremaining: 1.39s\n",
      "2:\tlearn: 0.9887746\ttotal: 40.4ms\tremaining: 1.3s\n",
      "3:\tlearn: 0.8708311\ttotal: 50.2ms\tremaining: 1.2s\n",
      "4:\tlearn: 0.7847531\ttotal: 59ms\tremaining: 1.12s\n",
      "5:\tlearn: 0.7022569\ttotal: 67.1ms\tremaining: 1.05s\n",
      "6:\tlearn: 0.6379313\ttotal: 73.7ms\tremaining: 979ms\n",
      "7:\tlearn: 0.5817027\ttotal: 80.3ms\tremaining: 923ms\n",
      "8:\tlearn: 0.5314632\ttotal: 86.7ms\tremaining: 877ms\n",
      "9:\tlearn: 0.4870568\ttotal: 92.5ms\tremaining: 832ms\n",
      "10:\tlearn: 0.4510508\ttotal: 98.1ms\tremaining: 794ms\n",
      "11:\tlearn: 0.4207161\ttotal: 103ms\tremaining: 757ms\n",
      "12:\tlearn: 0.3896809\ttotal: 109ms\tremaining: 728ms\n",
      "13:\tlearn: 0.3635224\ttotal: 114ms\tremaining: 700ms\n",
      "14:\tlearn: 0.3408099\ttotal: 119ms\tremaining: 676ms\n",
      "15:\tlearn: 0.3176504\ttotal: 125ms\tremaining: 657ms\n",
      "16:\tlearn: 0.2993085\ttotal: 130ms\tremaining: 636ms\n",
      "17:\tlearn: 0.2832547\ttotal: 136ms\tremaining: 618ms\n",
      "18:\tlearn: 0.2669124\ttotal: 141ms\tremaining: 602ms\n",
      "19:\tlearn: 0.2522774\ttotal: 147ms\tremaining: 587ms\n",
      "20:\tlearn: 0.2388176\ttotal: 152ms\tremaining: 572ms\n",
      "21:\tlearn: 0.2275164\ttotal: 157ms\tremaining: 558ms\n",
      "22:\tlearn: 0.2152957\ttotal: 163ms\tremaining: 546ms\n",
      "23:\tlearn: 0.2060060\ttotal: 168ms\tremaining: 533ms\n",
      "24:\tlearn: 0.1970027\ttotal: 174ms\tremaining: 521ms\n",
      "25:\tlearn: 0.1898256\ttotal: 179ms\tremaining: 508ms\n",
      "26:\tlearn: 0.1806685\ttotal: 184ms\tremaining: 498ms\n",
      "27:\tlearn: 0.1737680\ttotal: 189ms\tremaining: 486ms\n",
      "28:\tlearn: 0.1666664\ttotal: 194ms\tremaining: 475ms\n",
      "29:\tlearn: 0.1603577\ttotal: 200ms\tremaining: 466ms\n",
      "30:\tlearn: 0.1556248\ttotal: 204ms\tremaining: 455ms\n",
      "31:\tlearn: 0.1506939\ttotal: 209ms\tremaining: 445ms\n",
      "32:\tlearn: 0.1464950\ttotal: 214ms\tremaining: 435ms\n",
      "33:\tlearn: 0.1425808\ttotal: 219ms\tremaining: 426ms\n",
      "34:\tlearn: 0.1395360\ttotal: 224ms\tremaining: 417ms\n",
      "35:\tlearn: 0.1342278\ttotal: 229ms\tremaining: 408ms\n",
      "36:\tlearn: 0.1315332\ttotal: 234ms\tremaining: 399ms\n",
      "37:\tlearn: 0.1285900\ttotal: 240ms\tremaining: 391ms\n",
      "38:\tlearn: 0.1256996\ttotal: 244ms\tremaining: 382ms\n",
      "39:\tlearn: 0.1211779\ttotal: 250ms\tremaining: 375ms\n",
      "40:\tlearn: 0.1178496\ttotal: 255ms\tremaining: 367ms\n",
      "41:\tlearn: 0.1152184\ttotal: 260ms\tremaining: 359ms\n",
      "42:\tlearn: 0.1125398\ttotal: 265ms\tremaining: 351ms\n",
      "43:\tlearn: 0.1091040\ttotal: 270ms\tremaining: 344ms\n",
      "44:\tlearn: 0.1057295\ttotal: 276ms\tremaining: 337ms\n",
      "45:\tlearn: 0.1031605\ttotal: 280ms\tremaining: 329ms\n",
      "46:\tlearn: 0.1006224\ttotal: 286ms\tremaining: 322ms\n",
      "47:\tlearn: 0.0980683\ttotal: 291ms\tremaining: 315ms\n",
      "48:\tlearn: 0.0956914\ttotal: 296ms\tremaining: 308ms\n",
      "49:\tlearn: 0.0945311\ttotal: 300ms\tremaining: 300ms\n",
      "50:\tlearn: 0.0925995\ttotal: 305ms\tremaining: 293ms\n",
      "51:\tlearn: 0.0902052\ttotal: 310ms\tremaining: 287ms\n",
      "52:\tlearn: 0.0883770\ttotal: 315ms\tremaining: 280ms\n",
      "53:\tlearn: 0.0851485\ttotal: 321ms\tremaining: 273ms\n",
      "54:\tlearn: 0.0831049\ttotal: 326ms\tremaining: 266ms\n",
      "55:\tlearn: 0.0819903\ttotal: 331ms\tremaining: 260ms\n",
      "56:\tlearn: 0.0807019\ttotal: 335ms\tremaining: 253ms\n",
      "57:\tlearn: 0.0785594\ttotal: 341ms\tremaining: 247ms\n",
      "58:\tlearn: 0.0765932\ttotal: 346ms\tremaining: 240ms\n",
      "59:\tlearn: 0.0753131\ttotal: 351ms\tremaining: 234ms\n",
      "60:\tlearn: 0.0742942\ttotal: 356ms\tremaining: 228ms\n",
      "61:\tlearn: 0.0727772\ttotal: 361ms\tremaining: 221ms\n",
      "62:\tlearn: 0.0713396\ttotal: 366ms\tremaining: 215ms\n",
      "63:\tlearn: 0.0702139\ttotal: 370ms\tremaining: 208ms\n",
      "64:\tlearn: 0.0691017\ttotal: 375ms\tremaining: 202ms\n",
      "65:\tlearn: 0.0683416\ttotal: 380ms\tremaining: 196ms\n",
      "66:\tlearn: 0.0670372\ttotal: 385ms\tremaining: 190ms\n",
      "67:\tlearn: 0.0654676\ttotal: 390ms\tremaining: 184ms\n",
      "68:\tlearn: 0.0642326\ttotal: 396ms\tremaining: 178ms\n",
      "69:\tlearn: 0.0635505\ttotal: 400ms\tremaining: 172ms\n",
      "70:\tlearn: 0.0630354\ttotal: 405ms\tremaining: 165ms\n",
      "71:\tlearn: 0.0621747\ttotal: 411ms\tremaining: 160ms\n",
      "72:\tlearn: 0.0611664\ttotal: 416ms\tremaining: 154ms\n",
      "73:\tlearn: 0.0602606\ttotal: 421ms\tremaining: 148ms\n",
      "74:\tlearn: 0.0597102\ttotal: 426ms\tremaining: 142ms\n",
      "75:\tlearn: 0.0591920\ttotal: 430ms\tremaining: 136ms\n",
      "76:\tlearn: 0.0586500\ttotal: 435ms\tremaining: 130ms\n",
      "77:\tlearn: 0.0583329\ttotal: 440ms\tremaining: 124ms\n",
      "78:\tlearn: 0.0574854\ttotal: 445ms\tremaining: 118ms\n",
      "79:\tlearn: 0.0569879\ttotal: 451ms\tremaining: 113ms\n",
      "80:\tlearn: 0.0559364\ttotal: 456ms\tremaining: 107ms\n",
      "81:\tlearn: 0.0552718\ttotal: 461ms\tremaining: 101ms\n",
      "82:\tlearn: 0.0549534\ttotal: 465ms\tremaining: 95.3ms\n",
      "83:\tlearn: 0.0540290\ttotal: 471ms\tremaining: 89.7ms\n",
      "84:\tlearn: 0.0536795\ttotal: 475ms\tremaining: 83.9ms\n",
      "85:\tlearn: 0.0530022\ttotal: 480ms\tremaining: 78.2ms\n",
      "86:\tlearn: 0.0523051\ttotal: 485ms\tremaining: 72.5ms\n",
      "87:\tlearn: 0.0521336\ttotal: 490ms\tremaining: 66.8ms\n",
      "88:\tlearn: 0.0513894\ttotal: 495ms\tremaining: 61.1ms\n",
      "89:\tlearn: 0.0503908\ttotal: 500ms\tremaining: 55.5ms\n",
      "90:\tlearn: 0.0498218\ttotal: 505ms\tremaining: 49.9ms\n",
      "91:\tlearn: 0.0493108\ttotal: 509ms\tremaining: 44.3ms\n",
      "92:\tlearn: 0.0491307\ttotal: 514ms\tremaining: 38.7ms\n",
      "93:\tlearn: 0.0487834\ttotal: 519ms\tremaining: 33.1ms\n",
      "94:\tlearn: 0.0485043\ttotal: 524ms\tremaining: 27.6ms\n",
      "95:\tlearn: 0.0477113\ttotal: 529ms\tremaining: 22ms\n",
      "96:\tlearn: 0.0468495\ttotal: 534ms\tremaining: 16.5ms\n",
      "97:\tlearn: 0.0462060\ttotal: 539ms\tremaining: 11ms\n",
      "98:\tlearn: 0.0457864\ttotal: 544ms\tremaining: 5.5ms\n",
      "99:\tlearn: 0.0455426\ttotal: 549ms\tremaining: 0us\n",
      "0:\tlearn: 1.3335929\ttotal: 17.5ms\tremaining: 1.73s\n",
      "1:\tlearn: 1.1292976\ttotal: 30.5ms\tremaining: 1.49s\n",
      "2:\tlearn: 0.9822626\ttotal: 41.8ms\tremaining: 1.35s\n",
      "3:\tlearn: 0.8626856\ttotal: 53ms\tremaining: 1.27s\n",
      "4:\tlearn: 0.7738115\ttotal: 62ms\tremaining: 1.18s\n",
      "5:\tlearn: 0.6984786\ttotal: 70.4ms\tremaining: 1.1s\n",
      "6:\tlearn: 0.6366092\ttotal: 78.1ms\tremaining: 1.04s\n",
      "7:\tlearn: 0.5796810\ttotal: 84.9ms\tremaining: 977ms\n",
      "8:\tlearn: 0.5311805\ttotal: 91.5ms\tremaining: 925ms\n",
      "9:\tlearn: 0.4902507\ttotal: 97.6ms\tremaining: 878ms\n",
      "10:\tlearn: 0.4545794\ttotal: 104ms\tremaining: 840ms\n",
      "11:\tlearn: 0.4196416\ttotal: 110ms\tremaining: 805ms\n",
      "12:\tlearn: 0.3910011\ttotal: 116ms\tremaining: 774ms\n",
      "13:\tlearn: 0.3646772\ttotal: 121ms\tremaining: 744ms\n",
      "14:\tlearn: 0.3408263\ttotal: 127ms\tremaining: 718ms\n",
      "15:\tlearn: 0.3186315\ttotal: 133ms\tremaining: 696ms\n",
      "16:\tlearn: 0.2992084\ttotal: 138ms\tremaining: 676ms\n",
      "17:\tlearn: 0.2835010\ttotal: 144ms\tremaining: 655ms\n",
      "18:\tlearn: 0.2684400\ttotal: 150ms\tremaining: 638ms\n",
      "19:\tlearn: 0.2535621\ttotal: 155ms\tremaining: 620ms\n",
      "20:\tlearn: 0.2396000\ttotal: 160ms\tremaining: 604ms\n",
      "21:\tlearn: 0.2270608\ttotal: 166ms\tremaining: 588ms\n",
      "22:\tlearn: 0.2151384\ttotal: 171ms\tremaining: 573ms\n",
      "23:\tlearn: 0.2057487\ttotal: 176ms\tremaining: 558ms\n",
      "24:\tlearn: 0.1965170\ttotal: 182ms\tremaining: 546ms\n",
      "25:\tlearn: 0.1886311\ttotal: 187ms\tremaining: 533ms\n",
      "26:\tlearn: 0.1808337\ttotal: 192ms\tremaining: 520ms\n",
      "27:\tlearn: 0.1726177\ttotal: 197ms\tremaining: 508ms\n",
      "28:\tlearn: 0.1661025\ttotal: 202ms\tremaining: 495ms\n",
      "29:\tlearn: 0.1596358\ttotal: 208ms\tremaining: 485ms\n",
      "30:\tlearn: 0.1545877\ttotal: 212ms\tremaining: 473ms\n",
      "31:\tlearn: 0.1493745\ttotal: 218ms\tremaining: 463ms\n",
      "32:\tlearn: 0.1459648\ttotal: 223ms\tremaining: 453ms\n",
      "33:\tlearn: 0.1414937\ttotal: 228ms\tremaining: 443ms\n",
      "34:\tlearn: 0.1368273\ttotal: 233ms\tremaining: 433ms\n",
      "35:\tlearn: 0.1341699\ttotal: 238ms\tremaining: 423ms\n",
      "36:\tlearn: 0.1301503\ttotal: 243ms\tremaining: 414ms\n",
      "37:\tlearn: 0.1260273\ttotal: 248ms\tremaining: 405ms\n",
      "38:\tlearn: 0.1225161\ttotal: 253ms\tremaining: 396ms\n",
      "39:\tlearn: 0.1193672\ttotal: 258ms\tremaining: 387ms\n",
      "40:\tlearn: 0.1161301\ttotal: 263ms\tremaining: 378ms\n",
      "41:\tlearn: 0.1136074\ttotal: 268ms\tremaining: 370ms\n",
      "42:\tlearn: 0.1110420\ttotal: 273ms\tremaining: 362ms\n",
      "43:\tlearn: 0.1080385\ttotal: 278ms\tremaining: 354ms\n",
      "44:\tlearn: 0.1058469\ttotal: 283ms\tremaining: 346ms\n",
      "45:\tlearn: 0.1034568\ttotal: 288ms\tremaining: 338ms\n",
      "46:\tlearn: 0.0999953\ttotal: 293ms\tremaining: 331ms\n",
      "47:\tlearn: 0.0980002\ttotal: 298ms\tremaining: 323ms\n",
      "48:\tlearn: 0.0963940\ttotal: 304ms\tremaining: 316ms\n",
      "49:\tlearn: 0.0939995\ttotal: 309ms\tremaining: 309ms\n",
      "50:\tlearn: 0.0923570\ttotal: 313ms\tremaining: 301ms\n",
      "51:\tlearn: 0.0906639\ttotal: 318ms\tremaining: 294ms\n",
      "52:\tlearn: 0.0883851\ttotal: 323ms\tremaining: 287ms\n",
      "53:\tlearn: 0.0873517\ttotal: 328ms\tremaining: 279ms\n",
      "54:\tlearn: 0.0861761\ttotal: 333ms\tremaining: 272ms\n",
      "55:\tlearn: 0.0843029\ttotal: 337ms\tremaining: 265ms\n",
      "56:\tlearn: 0.0832444\ttotal: 342ms\tremaining: 258ms\n",
      "57:\tlearn: 0.0821903\ttotal: 347ms\tremaining: 251ms\n",
      "58:\tlearn: 0.0802455\ttotal: 352ms\tremaining: 245ms\n",
      "59:\tlearn: 0.0792245\ttotal: 357ms\tremaining: 238ms\n",
      "60:\tlearn: 0.0780307\ttotal: 362ms\tremaining: 231ms\n",
      "61:\tlearn: 0.0765510\ttotal: 367ms\tremaining: 225ms\n",
      "62:\tlearn: 0.0754388\ttotal: 372ms\tremaining: 218ms\n",
      "63:\tlearn: 0.0741115\ttotal: 376ms\tremaining: 212ms\n",
      "64:\tlearn: 0.0736596\ttotal: 381ms\tremaining: 205ms\n",
      "65:\tlearn: 0.0723215\ttotal: 386ms\tremaining: 199ms\n",
      "66:\tlearn: 0.0712819\ttotal: 390ms\tremaining: 192ms\n",
      "67:\tlearn: 0.0700027\ttotal: 395ms\tremaining: 186ms\n",
      "68:\tlearn: 0.0690495\ttotal: 400ms\tremaining: 180ms\n",
      "69:\tlearn: 0.0687463\ttotal: 404ms\tremaining: 173ms\n",
      "70:\tlearn: 0.0680955\ttotal: 409ms\tremaining: 167ms\n",
      "71:\tlearn: 0.0673044\ttotal: 414ms\tremaining: 161ms\n",
      "72:\tlearn: 0.0666110\ttotal: 419ms\tremaining: 155ms\n",
      "73:\tlearn: 0.0658388\ttotal: 424ms\tremaining: 149ms\n",
      "74:\tlearn: 0.0650439\ttotal: 429ms\tremaining: 143ms\n",
      "75:\tlearn: 0.0639647\ttotal: 434ms\tremaining: 137ms\n",
      "76:\tlearn: 0.0634520\ttotal: 439ms\tremaining: 131ms\n",
      "77:\tlearn: 0.0628246\ttotal: 443ms\tremaining: 125ms\n",
      "78:\tlearn: 0.0623357\ttotal: 448ms\tremaining: 119ms\n",
      "79:\tlearn: 0.0614606\ttotal: 453ms\tremaining: 113ms\n",
      "80:\tlearn: 0.0606629\ttotal: 459ms\tremaining: 108ms\n",
      "81:\tlearn: 0.0597263\ttotal: 464ms\tremaining: 102ms\n",
      "82:\tlearn: 0.0593305\ttotal: 468ms\tremaining: 95.9ms\n",
      "83:\tlearn: 0.0583714\ttotal: 473ms\tremaining: 90.2ms\n",
      "84:\tlearn: 0.0577446\ttotal: 478ms\tremaining: 84.4ms\n",
      "85:\tlearn: 0.0571000\ttotal: 483ms\tremaining: 78.6ms\n",
      "86:\tlearn: 0.0567583\ttotal: 488ms\tremaining: 72.9ms\n",
      "87:\tlearn: 0.0562680\ttotal: 493ms\tremaining: 67.2ms\n",
      "88:\tlearn: 0.0552306\ttotal: 498ms\tremaining: 61.5ms\n",
      "89:\tlearn: 0.0539584\ttotal: 503ms\tremaining: 55.9ms\n",
      "90:\tlearn: 0.0533775\ttotal: 508ms\tremaining: 50.2ms\n",
      "91:\tlearn: 0.0530248\ttotal: 512ms\tremaining: 44.5ms\n",
      "92:\tlearn: 0.0527591\ttotal: 517ms\tremaining: 38.9ms\n",
      "93:\tlearn: 0.0521129\ttotal: 522ms\tremaining: 33.3ms\n",
      "94:\tlearn: 0.0518451\ttotal: 526ms\tremaining: 27.7ms\n",
      "95:\tlearn: 0.0516507\ttotal: 531ms\tremaining: 22.1ms\n",
      "96:\tlearn: 0.0513344\ttotal: 536ms\tremaining: 16.6ms\n",
      "97:\tlearn: 0.0504281\ttotal: 541ms\tremaining: 11ms\n",
      "98:\tlearn: 0.0501084\ttotal: 546ms\tremaining: 5.51ms\n",
      "99:\tlearn: 0.0497951\ttotal: 550ms\tremaining: 0us\n",
      "0:\tlearn: 1.3434099\ttotal: 16.6ms\tremaining: 1.64s\n",
      "1:\tlearn: 1.1309979\ttotal: 28.2ms\tremaining: 1.38s\n",
      "2:\tlearn: 0.9968066\ttotal: 39.7ms\tremaining: 1.28s\n",
      "3:\tlearn: 0.8754597\ttotal: 49.2ms\tremaining: 1.18s\n",
      "4:\tlearn: 0.7844009\ttotal: 57.1ms\tremaining: 1.08s\n",
      "5:\tlearn: 0.7095462\ttotal: 64.6ms\tremaining: 1.01s\n",
      "6:\tlearn: 0.6422428\ttotal: 71.6ms\tremaining: 952ms\n",
      "7:\tlearn: 0.5843152\ttotal: 78.3ms\tremaining: 900ms\n",
      "8:\tlearn: 0.5337893\ttotal: 84.5ms\tremaining: 855ms\n",
      "9:\tlearn: 0.4923752\ttotal: 90ms\tremaining: 810ms\n",
      "10:\tlearn: 0.4569897\ttotal: 95.9ms\tremaining: 776ms\n",
      "11:\tlearn: 0.4268632\ttotal: 101ms\tremaining: 742ms\n",
      "12:\tlearn: 0.3959440\ttotal: 107ms\tremaining: 716ms\n",
      "13:\tlearn: 0.3687794\ttotal: 112ms\tremaining: 691ms\n",
      "14:\tlearn: 0.3475366\ttotal: 117ms\tremaining: 666ms\n",
      "15:\tlearn: 0.3252596\ttotal: 123ms\tremaining: 646ms\n",
      "16:\tlearn: 0.3075200\ttotal: 128ms\tremaining: 627ms\n",
      "17:\tlearn: 0.2892086\ttotal: 135ms\tremaining: 613ms\n",
      "18:\tlearn: 0.2724015\ttotal: 140ms\tremaining: 598ms\n",
      "19:\tlearn: 0.2577711\ttotal: 146ms\tremaining: 582ms\n",
      "20:\tlearn: 0.2455764\ttotal: 150ms\tremaining: 566ms\n",
      "21:\tlearn: 0.2330894\ttotal: 156ms\tremaining: 554ms\n",
      "22:\tlearn: 0.2221128\ttotal: 162ms\tremaining: 541ms\n",
      "23:\tlearn: 0.2121639\ttotal: 167ms\tremaining: 529ms\n",
      "24:\tlearn: 0.2015941\ttotal: 172ms\tremaining: 517ms\n",
      "25:\tlearn: 0.1942877\ttotal: 177ms\tremaining: 504ms\n",
      "26:\tlearn: 0.1850698\ttotal: 182ms\tremaining: 492ms\n",
      "27:\tlearn: 0.1764401\ttotal: 187ms\tremaining: 481ms\n",
      "28:\tlearn: 0.1706373\ttotal: 193ms\tremaining: 472ms\n",
      "29:\tlearn: 0.1641450\ttotal: 198ms\tremaining: 462ms\n",
      "30:\tlearn: 0.1588205\ttotal: 203ms\tremaining: 451ms\n",
      "31:\tlearn: 0.1533714\ttotal: 207ms\tremaining: 441ms\n",
      "32:\tlearn: 0.1490229\ttotal: 212ms\tremaining: 431ms\n",
      "33:\tlearn: 0.1445343\ttotal: 218ms\tremaining: 422ms\n",
      "34:\tlearn: 0.1395995\ttotal: 222ms\tremaining: 413ms\n",
      "35:\tlearn: 0.1341807\ttotal: 227ms\tremaining: 404ms\n",
      "36:\tlearn: 0.1296408\ttotal: 233ms\tremaining: 396ms\n",
      "37:\tlearn: 0.1254546\ttotal: 238ms\tremaining: 388ms\n",
      "38:\tlearn: 0.1212851\ttotal: 243ms\tremaining: 380ms\n",
      "39:\tlearn: 0.1173065\ttotal: 248ms\tremaining: 373ms\n",
      "40:\tlearn: 0.1143982\ttotal: 254ms\tremaining: 365ms\n",
      "41:\tlearn: 0.1117032\ttotal: 259ms\tremaining: 357ms\n",
      "42:\tlearn: 0.1086805\ttotal: 264ms\tremaining: 350ms\n",
      "43:\tlearn: 0.1047625\ttotal: 269ms\tremaining: 342ms\n",
      "44:\tlearn: 0.1022399\ttotal: 274ms\tremaining: 335ms\n",
      "45:\tlearn: 0.0993186\ttotal: 279ms\tremaining: 328ms\n",
      "46:\tlearn: 0.0968167\ttotal: 285ms\tremaining: 321ms\n",
      "47:\tlearn: 0.0942870\ttotal: 290ms\tremaining: 314ms\n",
      "48:\tlearn: 0.0927785\ttotal: 295ms\tremaining: 307ms\n",
      "49:\tlearn: 0.0904368\ttotal: 300ms\tremaining: 300ms\n",
      "50:\tlearn: 0.0889051\ttotal: 305ms\tremaining: 293ms\n",
      "51:\tlearn: 0.0869273\ttotal: 310ms\tremaining: 286ms\n",
      "52:\tlearn: 0.0850647\ttotal: 315ms\tremaining: 279ms\n",
      "53:\tlearn: 0.0836002\ttotal: 320ms\tremaining: 272ms\n",
      "54:\tlearn: 0.0825059\ttotal: 325ms\tremaining: 266ms\n",
      "55:\tlearn: 0.0812883\ttotal: 329ms\tremaining: 259ms\n",
      "56:\tlearn: 0.0795995\ttotal: 334ms\tremaining: 252ms\n",
      "57:\tlearn: 0.0784019\ttotal: 339ms\tremaining: 246ms\n",
      "58:\tlearn: 0.0770924\ttotal: 344ms\tremaining: 239ms\n",
      "59:\tlearn: 0.0753585\ttotal: 349ms\tremaining: 233ms\n",
      "60:\tlearn: 0.0745626\ttotal: 354ms\tremaining: 226ms\n",
      "61:\tlearn: 0.0739863\ttotal: 359ms\tremaining: 220ms\n",
      "62:\tlearn: 0.0730290\ttotal: 364ms\tremaining: 214ms\n",
      "63:\tlearn: 0.0715524\ttotal: 369ms\tremaining: 207ms\n",
      "64:\tlearn: 0.0707589\ttotal: 374ms\tremaining: 201ms\n",
      "65:\tlearn: 0.0696910\ttotal: 378ms\tremaining: 195ms\n",
      "66:\tlearn: 0.0683509\ttotal: 383ms\tremaining: 189ms\n",
      "67:\tlearn: 0.0663537\ttotal: 388ms\tremaining: 183ms\n",
      "68:\tlearn: 0.0651610\ttotal: 393ms\tremaining: 177ms\n",
      "69:\tlearn: 0.0644632\ttotal: 398ms\tremaining: 171ms\n",
      "70:\tlearn: 0.0641241\ttotal: 402ms\tremaining: 164ms\n",
      "71:\tlearn: 0.0632581\ttotal: 407ms\tremaining: 158ms\n",
      "72:\tlearn: 0.0621425\ttotal: 412ms\tremaining: 152ms\n",
      "73:\tlearn: 0.0613919\ttotal: 417ms\tremaining: 147ms\n",
      "74:\tlearn: 0.0609843\ttotal: 422ms\tremaining: 141ms\n",
      "75:\tlearn: 0.0602672\ttotal: 427ms\tremaining: 135ms\n",
      "76:\tlearn: 0.0597207\ttotal: 431ms\tremaining: 129ms\n",
      "77:\tlearn: 0.0590479\ttotal: 436ms\tremaining: 123ms\n",
      "78:\tlearn: 0.0583898\ttotal: 441ms\tremaining: 117ms\n",
      "79:\tlearn: 0.0583204\ttotal: 446ms\tremaining: 111ms\n",
      "80:\tlearn: 0.0572408\ttotal: 451ms\tremaining: 106ms\n",
      "81:\tlearn: 0.0561615\ttotal: 456ms\tremaining: 100ms\n",
      "82:\tlearn: 0.0556727\ttotal: 461ms\tremaining: 94.3ms\n",
      "83:\tlearn: 0.0548510\ttotal: 466ms\tremaining: 88.7ms\n",
      "84:\tlearn: 0.0545057\ttotal: 470ms\tremaining: 83ms\n",
      "85:\tlearn: 0.0537153\ttotal: 476ms\tremaining: 77.4ms\n",
      "86:\tlearn: 0.0531340\ttotal: 480ms\tremaining: 71.8ms\n",
      "87:\tlearn: 0.0530053\ttotal: 485ms\tremaining: 66.1ms\n",
      "88:\tlearn: 0.0524898\ttotal: 490ms\tremaining: 60.6ms\n",
      "89:\tlearn: 0.0517034\ttotal: 495ms\tremaining: 55ms\n",
      "90:\tlearn: 0.0509012\ttotal: 500ms\tremaining: 49.4ms\n",
      "91:\tlearn: 0.0503928\ttotal: 505ms\tremaining: 43.9ms\n",
      "92:\tlearn: 0.0501516\ttotal: 510ms\tremaining: 38.4ms\n",
      "93:\tlearn: 0.0495426\ttotal: 515ms\tremaining: 32.8ms\n",
      "94:\tlearn: 0.0489559\ttotal: 519ms\tremaining: 27.3ms\n",
      "95:\tlearn: 0.0485365\ttotal: 524ms\tremaining: 21.8ms\n",
      "96:\tlearn: 0.0480480\ttotal: 529ms\tremaining: 16.4ms\n",
      "97:\tlearn: 0.0476117\ttotal: 534ms\tremaining: 10.9ms\n",
      "98:\tlearn: 0.0469739\ttotal: 539ms\tremaining: 5.44ms\n",
      "99:\tlearn: 0.0469118\ttotal: 543ms\tremaining: 0us\n",
      "0:\tlearn: 1.3321443\ttotal: 17ms\tremaining: 1.68s\n",
      "1:\tlearn: 1.1289074\ttotal: 29ms\tremaining: 1.42s\n",
      "2:\tlearn: 0.9917005\ttotal: 41.2ms\tremaining: 1.33s\n",
      "3:\tlearn: 0.8693813\ttotal: 51.9ms\tremaining: 1.24s\n",
      "4:\tlearn: 0.7785982\ttotal: 61.2ms\tremaining: 1.16s\n",
      "5:\tlearn: 0.6969565\ttotal: 69.2ms\tremaining: 1.08s\n",
      "6:\tlearn: 0.6342302\ttotal: 76.9ms\tremaining: 1.02s\n",
      "7:\tlearn: 0.5770678\ttotal: 83.8ms\tremaining: 964ms\n",
      "8:\tlearn: 0.5276810\ttotal: 90.1ms\tremaining: 911ms\n",
      "9:\tlearn: 0.4854457\ttotal: 96.1ms\tremaining: 865ms\n",
      "10:\tlearn: 0.4493907\ttotal: 102ms\tremaining: 827ms\n",
      "11:\tlearn: 0.4193134\ttotal: 107ms\tremaining: 788ms\n",
      "12:\tlearn: 0.3897871\ttotal: 113ms\tremaining: 755ms\n",
      "13:\tlearn: 0.3625958\ttotal: 119ms\tremaining: 729ms\n",
      "14:\tlearn: 0.3386107\ttotal: 124ms\tremaining: 702ms\n",
      "15:\tlearn: 0.3174020\ttotal: 129ms\tremaining: 679ms\n",
      "16:\tlearn: 0.2978073\ttotal: 135ms\tremaining: 658ms\n",
      "17:\tlearn: 0.2798442\ttotal: 140ms\tremaining: 638ms\n",
      "18:\tlearn: 0.2626814\ttotal: 146ms\tremaining: 621ms\n",
      "19:\tlearn: 0.2477814\ttotal: 151ms\tremaining: 605ms\n",
      "20:\tlearn: 0.2343489\ttotal: 156ms\tremaining: 587ms\n",
      "21:\tlearn: 0.2230118\ttotal: 161ms\tremaining: 572ms\n",
      "22:\tlearn: 0.2125984\ttotal: 166ms\tremaining: 557ms\n",
      "23:\tlearn: 0.2037694\ttotal: 172ms\tremaining: 543ms\n",
      "24:\tlearn: 0.1937908\ttotal: 177ms\tremaining: 530ms\n",
      "25:\tlearn: 0.1854081\ttotal: 182ms\tremaining: 518ms\n",
      "26:\tlearn: 0.1765239\ttotal: 188ms\tremaining: 507ms\n",
      "27:\tlearn: 0.1679730\ttotal: 193ms\tremaining: 497ms\n",
      "28:\tlearn: 0.1620105\ttotal: 198ms\tremaining: 486ms\n",
      "29:\tlearn: 0.1548062\ttotal: 204ms\tremaining: 476ms\n",
      "30:\tlearn: 0.1485772\ttotal: 209ms\tremaining: 465ms\n",
      "31:\tlearn: 0.1443358\ttotal: 214ms\tremaining: 455ms\n",
      "32:\tlearn: 0.1395773\ttotal: 219ms\tremaining: 445ms\n",
      "33:\tlearn: 0.1355694\ttotal: 224ms\tremaining: 435ms\n",
      "34:\tlearn: 0.1315049\ttotal: 229ms\tremaining: 425ms\n",
      "35:\tlearn: 0.1272492\ttotal: 234ms\tremaining: 416ms\n",
      "36:\tlearn: 0.1233591\ttotal: 239ms\tremaining: 408ms\n",
      "37:\tlearn: 0.1191552\ttotal: 245ms\tremaining: 399ms\n",
      "38:\tlearn: 0.1147971\ttotal: 250ms\tremaining: 392ms\n",
      "39:\tlearn: 0.1113513\ttotal: 256ms\tremaining: 384ms\n",
      "40:\tlearn: 0.1092331\ttotal: 261ms\tremaining: 375ms\n",
      "41:\tlearn: 0.1057396\ttotal: 266ms\tremaining: 367ms\n",
      "42:\tlearn: 0.1028680\ttotal: 271ms\tremaining: 359ms\n",
      "43:\tlearn: 0.1004173\ttotal: 276ms\tremaining: 351ms\n",
      "44:\tlearn: 0.0988949\ttotal: 280ms\tremaining: 343ms\n",
      "45:\tlearn: 0.0967210\ttotal: 285ms\tremaining: 335ms\n",
      "46:\tlearn: 0.0943789\ttotal: 291ms\tremaining: 328ms\n",
      "47:\tlearn: 0.0921708\ttotal: 296ms\tremaining: 321ms\n",
      "48:\tlearn: 0.0901138\ttotal: 301ms\tremaining: 313ms\n",
      "49:\tlearn: 0.0885365\ttotal: 305ms\tremaining: 305ms\n",
      "50:\tlearn: 0.0866806\ttotal: 311ms\tremaining: 299ms\n",
      "51:\tlearn: 0.0856828\ttotal: 316ms\tremaining: 291ms\n",
      "52:\tlearn: 0.0838537\ttotal: 321ms\tremaining: 284ms\n",
      "53:\tlearn: 0.0823199\ttotal: 326ms\tremaining: 277ms\n",
      "54:\tlearn: 0.0805239\ttotal: 331ms\tremaining: 270ms\n",
      "55:\tlearn: 0.0788022\ttotal: 336ms\tremaining: 264ms\n",
      "56:\tlearn: 0.0772874\ttotal: 341ms\tremaining: 257ms\n",
      "57:\tlearn: 0.0756252\ttotal: 346ms\tremaining: 250ms\n",
      "58:\tlearn: 0.0749935\ttotal: 351ms\tremaining: 244ms\n",
      "59:\tlearn: 0.0734141\ttotal: 356ms\tremaining: 237ms\n",
      "60:\tlearn: 0.0725371\ttotal: 360ms\tremaining: 230ms\n",
      "61:\tlearn: 0.0713330\ttotal: 365ms\tremaining: 224ms\n",
      "62:\tlearn: 0.0700640\ttotal: 370ms\tremaining: 218ms\n",
      "63:\tlearn: 0.0690879\ttotal: 375ms\tremaining: 211ms\n",
      "64:\tlearn: 0.0681613\ttotal: 380ms\tremaining: 205ms\n",
      "65:\tlearn: 0.0672469\ttotal: 385ms\tremaining: 198ms\n",
      "66:\tlearn: 0.0665056\ttotal: 390ms\tremaining: 192ms\n",
      "67:\tlearn: 0.0658451\ttotal: 395ms\tremaining: 186ms\n",
      "68:\tlearn: 0.0649698\ttotal: 400ms\tremaining: 180ms\n",
      "69:\tlearn: 0.0645062\ttotal: 404ms\tremaining: 173ms\n",
      "70:\tlearn: 0.0635992\ttotal: 410ms\tremaining: 167ms\n",
      "71:\tlearn: 0.0623847\ttotal: 415ms\tremaining: 161ms\n",
      "72:\tlearn: 0.0614714\ttotal: 420ms\tremaining: 155ms\n",
      "73:\tlearn: 0.0609317\ttotal: 424ms\tremaining: 149ms\n",
      "74:\tlearn: 0.0603147\ttotal: 429ms\tremaining: 143ms\n",
      "75:\tlearn: 0.0598469\ttotal: 434ms\tremaining: 137ms\n",
      "76:\tlearn: 0.0590945\ttotal: 438ms\tremaining: 131ms\n",
      "77:\tlearn: 0.0586782\ttotal: 443ms\tremaining: 125ms\n",
      "78:\tlearn: 0.0581584\ttotal: 448ms\tremaining: 119ms\n",
      "79:\tlearn: 0.0573770\ttotal: 453ms\tremaining: 113ms\n",
      "80:\tlearn: 0.0563531\ttotal: 458ms\tremaining: 108ms\n",
      "81:\tlearn: 0.0556249\ttotal: 463ms\tremaining: 102ms\n",
      "82:\tlearn: 0.0546548\ttotal: 469ms\tremaining: 96ms\n",
      "83:\tlearn: 0.0540148\ttotal: 474ms\tremaining: 90.2ms\n",
      "84:\tlearn: 0.0536102\ttotal: 478ms\tremaining: 84.4ms\n",
      "85:\tlearn: 0.0532888\ttotal: 483ms\tremaining: 78.6ms\n",
      "86:\tlearn: 0.0525339\ttotal: 488ms\tremaining: 72.9ms\n",
      "87:\tlearn: 0.0520326\ttotal: 493ms\tremaining: 67.2ms\n",
      "88:\tlearn: 0.0513503\ttotal: 498ms\tremaining: 61.5ms\n",
      "89:\tlearn: 0.0508378\ttotal: 503ms\tremaining: 55.9ms\n",
      "90:\tlearn: 0.0502253\ttotal: 508ms\tremaining: 50.2ms\n",
      "91:\tlearn: 0.0495513\ttotal: 513ms\tremaining: 44.6ms\n",
      "92:\tlearn: 0.0491931\ttotal: 518ms\tremaining: 39ms\n",
      "93:\tlearn: 0.0485815\ttotal: 523ms\tremaining: 33.4ms\n",
      "94:\tlearn: 0.0481127\ttotal: 527ms\tremaining: 27.8ms\n",
      "95:\tlearn: 0.0476646\ttotal: 532ms\tremaining: 22.2ms\n",
      "96:\tlearn: 0.0474917\ttotal: 537ms\tremaining: 16.6ms\n",
      "97:\tlearn: 0.0471671\ttotal: 541ms\tremaining: 11ms\n",
      "98:\tlearn: 0.0466096\ttotal: 546ms\tremaining: 5.52ms\n",
      "99:\tlearn: 0.0462499\ttotal: 551ms\tremaining: 0us\n",
      "0:\tlearn: 1.3238347\ttotal: 17.5ms\tremaining: 1.73s\n",
      "1:\tlearn: 1.1173513\ttotal: 30.7ms\tremaining: 1.5s\n",
      "2:\tlearn: 0.9845297\ttotal: 43.3ms\tremaining: 1.4s\n",
      "3:\tlearn: 0.8658608\ttotal: 53.5ms\tremaining: 1.28s\n",
      "4:\tlearn: 0.7778703\ttotal: 63ms\tremaining: 1.2s\n",
      "5:\tlearn: 0.7027518\ttotal: 71.3ms\tremaining: 1.12s\n",
      "6:\tlearn: 0.6387611\ttotal: 78.9ms\tremaining: 1.05s\n",
      "7:\tlearn: 0.5795509\ttotal: 86.1ms\tremaining: 990ms\n",
      "8:\tlearn: 0.5287721\ttotal: 92.3ms\tremaining: 933ms\n",
      "9:\tlearn: 0.4864047\ttotal: 98.4ms\tremaining: 885ms\n",
      "10:\tlearn: 0.4513530\ttotal: 104ms\tremaining: 841ms\n",
      "11:\tlearn: 0.4182467\ttotal: 109ms\tremaining: 803ms\n",
      "12:\tlearn: 0.3883430\ttotal: 115ms\tremaining: 770ms\n",
      "13:\tlearn: 0.3614481\ttotal: 120ms\tremaining: 740ms\n",
      "14:\tlearn: 0.3392478\ttotal: 126ms\tremaining: 713ms\n",
      "15:\tlearn: 0.3163891\ttotal: 131ms\tremaining: 689ms\n",
      "16:\tlearn: 0.2968557\ttotal: 136ms\tremaining: 665ms\n",
      "17:\tlearn: 0.2792251\ttotal: 141ms\tremaining: 645ms\n",
      "18:\tlearn: 0.2637954\ttotal: 147ms\tremaining: 628ms\n",
      "19:\tlearn: 0.2479766\ttotal: 153ms\tremaining: 611ms\n",
      "20:\tlearn: 0.2348442\ttotal: 158ms\tremaining: 594ms\n",
      "21:\tlearn: 0.2228533\ttotal: 163ms\tremaining: 578ms\n",
      "22:\tlearn: 0.2123306\ttotal: 168ms\tremaining: 564ms\n",
      "23:\tlearn: 0.2033978\ttotal: 173ms\tremaining: 549ms\n",
      "24:\tlearn: 0.1943759\ttotal: 179ms\tremaining: 537ms\n",
      "25:\tlearn: 0.1870514\ttotal: 184ms\tremaining: 524ms\n",
      "26:\tlearn: 0.1785219\ttotal: 189ms\tremaining: 511ms\n",
      "27:\tlearn: 0.1715675\ttotal: 194ms\tremaining: 500ms\n",
      "28:\tlearn: 0.1653232\ttotal: 199ms\tremaining: 488ms\n",
      "29:\tlearn: 0.1595955\ttotal: 204ms\tremaining: 477ms\n",
      "30:\tlearn: 0.1546688\ttotal: 209ms\tremaining: 466ms\n",
      "31:\tlearn: 0.1491627\ttotal: 215ms\tremaining: 456ms\n",
      "32:\tlearn: 0.1457602\ttotal: 220ms\tremaining: 446ms\n",
      "33:\tlearn: 0.1412249\ttotal: 225ms\tremaining: 437ms\n",
      "34:\tlearn: 0.1369450\ttotal: 230ms\tremaining: 427ms\n",
      "35:\tlearn: 0.1326888\ttotal: 235ms\tremaining: 418ms\n",
      "36:\tlearn: 0.1285851\ttotal: 240ms\tremaining: 409ms\n",
      "37:\tlearn: 0.1237387\ttotal: 245ms\tremaining: 400ms\n",
      "38:\tlearn: 0.1202933\ttotal: 250ms\tremaining: 392ms\n",
      "39:\tlearn: 0.1162623\ttotal: 256ms\tremaining: 383ms\n",
      "40:\tlearn: 0.1131791\ttotal: 261ms\tremaining: 375ms\n",
      "41:\tlearn: 0.1090991\ttotal: 266ms\tremaining: 367ms\n",
      "42:\tlearn: 0.1069087\ttotal: 271ms\tremaining: 359ms\n",
      "43:\tlearn: 0.1040861\ttotal: 276ms\tremaining: 351ms\n",
      "44:\tlearn: 0.1025038\ttotal: 281ms\tremaining: 343ms\n",
      "45:\tlearn: 0.0996577\ttotal: 286ms\tremaining: 335ms\n",
      "46:\tlearn: 0.0971710\ttotal: 291ms\tremaining: 328ms\n",
      "47:\tlearn: 0.0950524\ttotal: 296ms\tremaining: 321ms\n",
      "48:\tlearn: 0.0934152\ttotal: 301ms\tremaining: 313ms\n",
      "49:\tlearn: 0.0912342\ttotal: 306ms\tremaining: 306ms\n",
      "50:\tlearn: 0.0899739\ttotal: 311ms\tremaining: 299ms\n",
      "51:\tlearn: 0.0874127\ttotal: 316ms\tremaining: 292ms\n",
      "52:\tlearn: 0.0858967\ttotal: 321ms\tremaining: 285ms\n",
      "53:\tlearn: 0.0846762\ttotal: 326ms\tremaining: 278ms\n",
      "54:\tlearn: 0.0833845\ttotal: 331ms\tremaining: 270ms\n",
      "55:\tlearn: 0.0818102\ttotal: 335ms\tremaining: 263ms\n",
      "56:\tlearn: 0.0805269\ttotal: 340ms\tremaining: 256ms\n",
      "57:\tlearn: 0.0798128\ttotal: 345ms\tremaining: 250ms\n",
      "58:\tlearn: 0.0780496\ttotal: 350ms\tremaining: 243ms\n",
      "59:\tlearn: 0.0767524\ttotal: 354ms\tremaining: 236ms\n",
      "60:\tlearn: 0.0758843\ttotal: 359ms\tremaining: 230ms\n",
      "61:\tlearn: 0.0753931\ttotal: 364ms\tremaining: 223ms\n",
      "62:\tlearn: 0.0748269\ttotal: 369ms\tremaining: 216ms\n",
      "63:\tlearn: 0.0739214\ttotal: 374ms\tremaining: 210ms\n",
      "64:\tlearn: 0.0727399\ttotal: 379ms\tremaining: 204ms\n",
      "65:\tlearn: 0.0724661\ttotal: 384ms\tremaining: 198ms\n",
      "66:\tlearn: 0.0712920\ttotal: 389ms\tremaining: 191ms\n",
      "67:\tlearn: 0.0697861\ttotal: 393ms\tremaining: 185ms\n",
      "68:\tlearn: 0.0688911\ttotal: 399ms\tremaining: 179ms\n",
      "69:\tlearn: 0.0682388\ttotal: 404ms\tremaining: 173ms\n",
      "70:\tlearn: 0.0674411\ttotal: 408ms\tremaining: 167ms\n",
      "71:\tlearn: 0.0667707\ttotal: 414ms\tremaining: 161ms\n",
      "72:\tlearn: 0.0659356\ttotal: 418ms\tremaining: 155ms\n",
      "73:\tlearn: 0.0650213\ttotal: 423ms\tremaining: 149ms\n",
      "74:\tlearn: 0.0647043\ttotal: 428ms\tremaining: 143ms\n",
      "75:\tlearn: 0.0640779\ttotal: 433ms\tremaining: 137ms\n",
      "76:\tlearn: 0.0629325\ttotal: 438ms\tremaining: 131ms\n",
      "77:\tlearn: 0.0623778\ttotal: 442ms\tremaining: 125ms\n",
      "78:\tlearn: 0.0614266\ttotal: 447ms\tremaining: 119ms\n",
      "79:\tlearn: 0.0608154\ttotal: 452ms\tremaining: 113ms\n",
      "80:\tlearn: 0.0603216\ttotal: 457ms\tremaining: 107ms\n",
      "81:\tlearn: 0.0599934\ttotal: 461ms\tremaining: 101ms\n",
      "82:\tlearn: 0.0594610\ttotal: 466ms\tremaining: 95.4ms\n",
      "83:\tlearn: 0.0590640\ttotal: 471ms\tremaining: 89.7ms\n",
      "84:\tlearn: 0.0586688\ttotal: 476ms\tremaining: 84ms\n",
      "85:\tlearn: 0.0577816\ttotal: 481ms\tremaining: 78.3ms\n",
      "86:\tlearn: 0.0569078\ttotal: 486ms\tremaining: 72.6ms\n",
      "87:\tlearn: 0.0561565\ttotal: 491ms\tremaining: 66.9ms\n",
      "88:\tlearn: 0.0558290\ttotal: 495ms\tremaining: 61.2ms\n",
      "89:\tlearn: 0.0548910\ttotal: 501ms\tremaining: 55.6ms\n",
      "90:\tlearn: 0.0543410\ttotal: 505ms\tremaining: 50ms\n",
      "91:\tlearn: 0.0537224\ttotal: 510ms\tremaining: 44.4ms\n",
      "92:\tlearn: 0.0532689\ttotal: 515ms\tremaining: 38.8ms\n",
      "93:\tlearn: 0.0527524\ttotal: 520ms\tremaining: 33.2ms\n",
      "94:\tlearn: 0.0522019\ttotal: 525ms\tremaining: 27.7ms\n",
      "95:\tlearn: 0.0514369\ttotal: 531ms\tremaining: 22.1ms\n",
      "96:\tlearn: 0.0510239\ttotal: 535ms\tremaining: 16.6ms\n",
      "97:\tlearn: 0.0504051\ttotal: 540ms\tremaining: 11ms\n",
      "98:\tlearn: 0.0499257\ttotal: 545ms\tremaining: 5.51ms\n",
      "99:\tlearn: 0.0497038\ttotal: 550ms\tremaining: 0us\n",
      "0:\tlearn: 1.3343878\ttotal: 16.9ms\tremaining: 1.67s\n",
      "1:\tlearn: 1.1312489\ttotal: 29.7ms\tremaining: 1.45s\n",
      "2:\tlearn: 0.9966622\ttotal: 41.7ms\tremaining: 1.35s\n",
      "3:\tlearn: 0.8752126\ttotal: 51.5ms\tremaining: 1.24s\n",
      "4:\tlearn: 0.7857153\ttotal: 60.4ms\tremaining: 1.15s\n",
      "5:\tlearn: 0.7045660\ttotal: 67.9ms\tremaining: 1.06s\n",
      "6:\tlearn: 0.6403210\ttotal: 75.4ms\tremaining: 1s\n",
      "7:\tlearn: 0.5825343\ttotal: 82.2ms\tremaining: 945ms\n",
      "8:\tlearn: 0.5353488\ttotal: 88.1ms\tremaining: 891ms\n",
      "9:\tlearn: 0.4921871\ttotal: 94ms\tremaining: 846ms\n",
      "10:\tlearn: 0.4568478\ttotal: 99.7ms\tremaining: 806ms\n",
      "11:\tlearn: 0.4226686\ttotal: 105ms\tremaining: 773ms\n",
      "12:\tlearn: 0.3930197\ttotal: 111ms\tremaining: 742ms\n",
      "13:\tlearn: 0.3658434\ttotal: 116ms\tremaining: 714ms\n",
      "14:\tlearn: 0.3425489\ttotal: 121ms\tremaining: 688ms\n",
      "15:\tlearn: 0.3202487\ttotal: 127ms\tremaining: 666ms\n",
      "16:\tlearn: 0.3017371\ttotal: 132ms\tremaining: 645ms\n",
      "17:\tlearn: 0.2843290\ttotal: 137ms\tremaining: 625ms\n",
      "18:\tlearn: 0.2683837\ttotal: 143ms\tremaining: 609ms\n",
      "19:\tlearn: 0.2554026\ttotal: 148ms\tremaining: 592ms\n",
      "20:\tlearn: 0.2424006\ttotal: 153ms\tremaining: 576ms\n",
      "21:\tlearn: 0.2292297\ttotal: 158ms\tremaining: 561ms\n",
      "22:\tlearn: 0.2182467\ttotal: 164ms\tremaining: 549ms\n",
      "23:\tlearn: 0.2078820\ttotal: 169ms\tremaining: 536ms\n",
      "24:\tlearn: 0.1986061\ttotal: 174ms\tremaining: 523ms\n",
      "25:\tlearn: 0.1910144\ttotal: 179ms\tremaining: 510ms\n",
      "26:\tlearn: 0.1823769\ttotal: 184ms\tremaining: 499ms\n",
      "27:\tlearn: 0.1740764\ttotal: 190ms\tremaining: 487ms\n",
      "28:\tlearn: 0.1682643\ttotal: 195ms\tremaining: 477ms\n",
      "29:\tlearn: 0.1632803\ttotal: 200ms\tremaining: 466ms\n",
      "30:\tlearn: 0.1579528\ttotal: 205ms\tremaining: 456ms\n",
      "31:\tlearn: 0.1530852\ttotal: 210ms\tremaining: 446ms\n",
      "32:\tlearn: 0.1483302\ttotal: 215ms\tremaining: 436ms\n",
      "33:\tlearn: 0.1433344\ttotal: 220ms\tremaining: 427ms\n",
      "34:\tlearn: 0.1394629\ttotal: 225ms\tremaining: 418ms\n",
      "35:\tlearn: 0.1357931\ttotal: 230ms\tremaining: 409ms\n",
      "36:\tlearn: 0.1320789\ttotal: 235ms\tremaining: 400ms\n",
      "37:\tlearn: 0.1288770\ttotal: 240ms\tremaining: 392ms\n",
      "38:\tlearn: 0.1250727\ttotal: 246ms\tremaining: 384ms\n",
      "39:\tlearn: 0.1216006\ttotal: 251ms\tremaining: 376ms\n",
      "40:\tlearn: 0.1184990\ttotal: 256ms\tremaining: 369ms\n",
      "41:\tlearn: 0.1136620\ttotal: 261ms\tremaining: 361ms\n",
      "42:\tlearn: 0.1113136\ttotal: 266ms\tremaining: 353ms\n",
      "43:\tlearn: 0.1081127\ttotal: 271ms\tremaining: 345ms\n",
      "44:\tlearn: 0.1058612\ttotal: 276ms\tremaining: 337ms\n",
      "45:\tlearn: 0.1029561\ttotal: 281ms\tremaining: 330ms\n",
      "46:\tlearn: 0.1018269\ttotal: 285ms\tremaining: 322ms\n",
      "47:\tlearn: 0.0992377\ttotal: 291ms\tremaining: 315ms\n",
      "48:\tlearn: 0.0971726\ttotal: 296ms\tremaining: 308ms\n",
      "49:\tlearn: 0.0952461\ttotal: 301ms\tremaining: 301ms\n",
      "50:\tlearn: 0.0939431\ttotal: 306ms\tremaining: 294ms\n",
      "51:\tlearn: 0.0924726\ttotal: 310ms\tremaining: 286ms\n",
      "52:\tlearn: 0.0899822\ttotal: 315ms\tremaining: 279ms\n",
      "53:\tlearn: 0.0886806\ttotal: 320ms\tremaining: 273ms\n",
      "54:\tlearn: 0.0866686\ttotal: 325ms\tremaining: 266ms\n",
      "55:\tlearn: 0.0853865\ttotal: 330ms\tremaining: 259ms\n",
      "56:\tlearn: 0.0834835\ttotal: 335ms\tremaining: 253ms\n",
      "57:\tlearn: 0.0824699\ttotal: 340ms\tremaining: 246ms\n",
      "58:\tlearn: 0.0812637\ttotal: 344ms\tremaining: 239ms\n",
      "59:\tlearn: 0.0793652\ttotal: 349ms\tremaining: 233ms\n",
      "60:\tlearn: 0.0786018\ttotal: 354ms\tremaining: 226ms\n",
      "61:\tlearn: 0.0773920\ttotal: 359ms\tremaining: 220ms\n",
      "62:\tlearn: 0.0752403\ttotal: 364ms\tremaining: 214ms\n",
      "63:\tlearn: 0.0742251\ttotal: 369ms\tremaining: 208ms\n",
      "64:\tlearn: 0.0730544\ttotal: 374ms\tremaining: 201ms\n",
      "65:\tlearn: 0.0719155\ttotal: 379ms\tremaining: 195ms\n",
      "66:\tlearn: 0.0703472\ttotal: 384ms\tremaining: 189ms\n",
      "67:\tlearn: 0.0693851\ttotal: 389ms\tremaining: 183ms\n",
      "68:\tlearn: 0.0683955\ttotal: 394ms\tremaining: 177ms\n",
      "69:\tlearn: 0.0675036\ttotal: 398ms\tremaining: 171ms\n",
      "70:\tlearn: 0.0662913\ttotal: 404ms\tremaining: 165ms\n",
      "71:\tlearn: 0.0657696\ttotal: 408ms\tremaining: 159ms\n",
      "72:\tlearn: 0.0650054\ttotal: 414ms\tremaining: 153ms\n",
      "73:\tlearn: 0.0642117\ttotal: 418ms\tremaining: 147ms\n",
      "74:\tlearn: 0.0635361\ttotal: 423ms\tremaining: 141ms\n",
      "75:\tlearn: 0.0628358\ttotal: 428ms\tremaining: 135ms\n",
      "76:\tlearn: 0.0623203\ttotal: 433ms\tremaining: 129ms\n",
      "77:\tlearn: 0.0619385\ttotal: 438ms\tremaining: 123ms\n",
      "78:\tlearn: 0.0616132\ttotal: 442ms\tremaining: 118ms\n",
      "79:\tlearn: 0.0610394\ttotal: 447ms\tremaining: 112ms\n",
      "80:\tlearn: 0.0602350\ttotal: 452ms\tremaining: 106ms\n",
      "81:\tlearn: 0.0596090\ttotal: 457ms\tremaining: 100ms\n",
      "82:\tlearn: 0.0588651\ttotal: 462ms\tremaining: 94.6ms\n",
      "83:\tlearn: 0.0579919\ttotal: 466ms\tremaining: 88.8ms\n",
      "84:\tlearn: 0.0570446\ttotal: 471ms\tremaining: 83.2ms\n",
      "85:\tlearn: 0.0561851\ttotal: 476ms\tremaining: 77.5ms\n",
      "86:\tlearn: 0.0555583\ttotal: 481ms\tremaining: 71.9ms\n",
      "87:\tlearn: 0.0547111\ttotal: 486ms\tremaining: 66.3ms\n",
      "88:\tlearn: 0.0544033\ttotal: 491ms\tremaining: 60.7ms\n",
      "89:\tlearn: 0.0535293\ttotal: 496ms\tremaining: 55.1ms\n",
      "90:\tlearn: 0.0527254\ttotal: 500ms\tremaining: 49.5ms\n",
      "91:\tlearn: 0.0521690\ttotal: 505ms\tremaining: 43.9ms\n",
      "92:\tlearn: 0.0519258\ttotal: 510ms\tremaining: 38.4ms\n",
      "93:\tlearn: 0.0512047\ttotal: 515ms\tremaining: 32.9ms\n",
      "94:\tlearn: 0.0503773\ttotal: 520ms\tremaining: 27.4ms\n",
      "95:\tlearn: 0.0497840\ttotal: 525ms\tremaining: 21.9ms\n",
      "96:\tlearn: 0.0490498\ttotal: 530ms\tremaining: 16.4ms\n",
      "97:\tlearn: 0.0486444\ttotal: 535ms\tremaining: 10.9ms\n",
      "98:\tlearn: 0.0479239\ttotal: 540ms\tremaining: 5.46ms\n",
      "99:\tlearn: 0.0471950\ttotal: 545ms\tremaining: 0us\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2      3    4\n",
      "0  2349.0     5.0    4.0    9.0  0.0\n",
      "1     7.0  1596.0    0.0    0.0  0.0\n",
      "2    19.0     3.0  383.0    1.0  0.0\n",
      "3     8.0     0.0    2.0  113.0  0.0\n",
      "4     1.0     0.0    0.0    0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9868888888888889\n",
      "Precision total:  0.7767212566483375\n",
      "Recall total:  0.7700155131443468\n",
      "F1 total:  0.7732776769788952\n",
      "BACC total:  0.7700155131443468\n",
      "MCC total:  0.977647317043085\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "start = time.time()\n",
    "\n",
    "bag_cat = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "base_classifier = bag_cat\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_cat'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test, time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging Combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3047254\ttest: 1.3016224\tbest: 1.3016224 (0)\ttotal: 19.2ms\tremaining: 1.9s\n",
      "10:\tlearn: 0.4533872\ttest: 0.4514314\tbest: 0.4514314 (10)\ttotal: 123ms\tremaining: 995ms\n",
      "20:\tlearn: 0.2339763\ttest: 0.2357504\tbest: 0.2357504 (20)\ttotal: 181ms\tremaining: 682ms\n",
      "30:\tlearn: 0.1480733\ttest: 0.1512366\tbest: 0.1512366 (30)\ttotal: 234ms\tremaining: 520ms\n",
      "40:\tlearn: 0.1124202\ttest: 0.1179599\tbest: 0.1179599 (40)\ttotal: 282ms\tremaining: 406ms\n",
      "50:\tlearn: 0.0891353\ttest: 0.0962047\tbest: 0.0962047 (50)\ttotal: 332ms\tremaining: 319ms\n",
      "60:\tlearn: 0.0740737\ttest: 0.0820106\tbest: 0.0820106 (60)\ttotal: 380ms\tremaining: 243ms\n",
      "70:\tlearn: 0.0652873\ttest: 0.0740704\tbest: 0.0740704 (70)\ttotal: 428ms\tremaining: 175ms\n",
      "80:\tlearn: 0.0573126\ttest: 0.0671240\tbest: 0.0671240 (80)\ttotal: 478ms\tremaining: 112ms\n",
      "90:\tlearn: 0.0517011\ttest: 0.0618311\tbest: 0.0618311 (90)\ttotal: 526ms\tremaining: 52.1ms\n",
      "99:\tlearn: 0.0476242\ttest: 0.0581125\tbest: 0.0581125 (99)\ttotal: 570ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.05811252376\n",
      "bestIteration = 99\n",
      "\n",
      "Epoch 1/100\n",
      "66/66 [==============================] - 1s 5ms/step - loss: 2.1549 - accuracy: 0.2754 - val_loss: 1.5396 - val_accuracy: 0.5538\n",
      "Epoch 2/100\n",
      "66/66 [==============================] - 0s 3ms/step - loss: 1.5095 - accuracy: 0.4750 - val_loss: 1.2897 - val_accuracy: 0.7043\n",
      "Epoch 3/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 1.2972 - accuracy: 0.5651 - val_loss: 1.0522 - val_accuracy: 0.8262\n",
      "Epoch 4/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 1.1180 - accuracy: 0.6405 - val_loss: 0.8261 - val_accuracy: 0.8381\n",
      "Epoch 5/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.9792 - accuracy: 0.6907 - val_loss: 0.6701 - val_accuracy: 0.8438\n",
      "Epoch 6/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.8717 - accuracy: 0.7220 - val_loss: 0.5749 - val_accuracy: 0.8457\n",
      "Epoch 7/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.8032 - accuracy: 0.7675 - val_loss: 0.5202 - val_accuracy: 0.8471\n",
      "Epoch 8/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.7429 - accuracy: 0.7821 - val_loss: 0.4890 - val_accuracy: 0.8481\n",
      "Epoch 9/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.7066 - accuracy: 0.7933 - val_loss: 0.4688 - val_accuracy: 0.8490\n",
      "Epoch 10/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.6576 - accuracy: 0.8044 - val_loss: 0.4525 - val_accuracy: 0.8462\n",
      "Epoch 11/100\n",
      "66/66 [==============================] - 0s 3ms/step - loss: 0.6542 - accuracy: 0.8154 - val_loss: 0.4390 - val_accuracy: 0.8462\n",
      "Epoch 12/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.6184 - accuracy: 0.8182 - val_loss: 0.4306 - val_accuracy: 0.8467\n",
      "Epoch 13/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5977 - accuracy: 0.8251 - val_loss: 0.4202 - val_accuracy: 0.8471\n",
      "Epoch 14/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5923 - accuracy: 0.8227 - val_loss: 0.4131 - val_accuracy: 0.8467\n",
      "Epoch 15/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5637 - accuracy: 0.8288 - val_loss: 0.4033 - val_accuracy: 0.8462\n",
      "Epoch 16/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5360 - accuracy: 0.8333 - val_loss: 0.3984 - val_accuracy: 0.8462\n",
      "Epoch 17/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5268 - accuracy: 0.8356 - val_loss: 0.3909 - val_accuracy: 0.8462\n",
      "Epoch 18/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5139 - accuracy: 0.8355 - val_loss: 0.3855 - val_accuracy: 0.8481\n",
      "Epoch 19/100\n",
      "66/66 [==============================] - 0s 4ms/step - loss: 0.5087 - accuracy: 0.8339 - val_loss: 0.3780 - val_accuracy: 0.8481\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0           1        2        1        1        1        1        1        1   \n",
      "1           0        0        0        0        0        0        0        0   \n",
      "2           1        2        1        1        1        1        1        1   \n",
      "3           0        0        0        0        1        0        0        0   \n",
      "4           1        2        2        1        2        1        1        1   \n",
      "...       ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "4495        0        1        0        0        0        0        0        0   \n",
      "4496        1        2        1        1        1        1        1        1   \n",
      "4497        1        2        1        1        1        1        1        1   \n",
      "4498        0        0        0        0        0        0        0        0   \n",
      "4499        0        0        0        0        0        0        0        0   \n",
      "\n",
      "      model_8  model_9  \n",
      "0           1      1.0  \n",
      "1           0      0.0  \n",
      "2           1      1.0  \n",
      "3           0      0.0  \n",
      "4           1      2.0  \n",
      "...       ...      ...  \n",
      "4495        0      0.0  \n",
      "4496        1      1.0  \n",
      "4497        1      1.0  \n",
      "4498        0      0.0  \n",
      "4499        0      0.0  \n",
      "\n",
      "[4500 rows x 10 columns]\n",
      "      model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0           1        2        1        1        1        1        1        1   \n",
      "1           0        0        0        0        0        0        0        0   \n",
      "2           1        2        1        1        1        1        1        1   \n",
      "3           0        0        0        0        1        0        0        0   \n",
      "4           1        2        2        1        2        1        1        1   \n",
      "...       ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "4495        0        1        0        0        0        0        0        0   \n",
      "4496        1        2        1        1        1        1        1        1   \n",
      "4497        1        2        1        1        1        1        1        1   \n",
      "4498        0        0        0        0        0        0        0        0   \n",
      "4499        0        0        0        0        0        0        0        0   \n",
      "\n",
      "      model_8  model_9  ensemble  \n",
      "0           1      1.0         1  \n",
      "1           0      0.0         0  \n",
      "2           1      1.0         1  \n",
      "3           0      0.0         0  \n",
      "4           1      2.0         1  \n",
      "...       ...      ...       ...  \n",
      "4495        0      0.0         0  \n",
      "4496        1      1.0         1  \n",
      "4497        1      1.0         1  \n",
      "4498        0      0.0         0  \n",
      "4499        0      0.0         0  \n",
      "\n",
      "[4500 rows x 11 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0       1      2     3    4\n",
      "0  2355.0     9.0    0.0   3.0  0.0\n",
      "1    23.0  1580.0    0.0   0.0  0.0\n",
      "2    25.0    10.0  371.0   0.0  0.0\n",
      "3    39.0     0.0    0.0  84.0  0.0\n",
      "4     1.0     0.0    0.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9755555555555555\n",
      "Precision total:  0.7835227059115569\n",
      "Recall total:  0.7154604253814555\n",
      "F1 total:  0.7442096264332367\n",
      "BACC total:  0.7154604253814555\n",
      "MCC total:  0.9582739566461257\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "### Bagging with many models\n",
    "##### do bootstrapping \n",
    "##### 1. Multiple subsets are created from the original dataset, selecting observations with replacement.\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "num_bootstraps = 10  # Adjust the number of bootstraps as needed\n",
    "\n",
    "original_data_df = X_train.assign(label = y_train)\n",
    "boot_df = []\n",
    "for i in range(0,num_bootstraps): \n",
    "    boot_df.append(original_data_df.sample(frac = 1, replace=True).reset_index(drop=True))\n",
    "\n",
    "# boot_df[5]\n",
    "\n",
    "#### 2.A base model (weak model) is created on each of these subsets.\n",
    "bag_comb_pred = []\n",
    "\n",
    "# SVM\n",
    "\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "y_train_boot = boot_df[0].pop('label')\n",
    "X_train_boot = boot_df[0]\n",
    "clf.fit(X_train_boot, y_train_boot)\n",
    "preds_svm_00 = clf.predict(X_test)\n",
    "bag_comb_pred.append(preds_svm_00)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#ADA\n",
    "\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "ada = abc.fit(X_train, y_train)\n",
    "y_train_boot = boot_df[1].pop('label')\n",
    "X_train_boot = boot_df[1]\n",
    "preds_ada_00 = ada.predict(X_test)\n",
    "bag_comb_pred.append(preds_ada_00)\n",
    "\n",
    "#Catboost\n",
    "\n",
    "cat_00 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "y_train_boot = boot_df[2].pop('label')\n",
    "X_train_boot = boot_df[2]\n",
    "cat_00.fit(X_train_boot, y_train_boot, eval_set=(X_test, y_test), verbose=10)\n",
    "preds_cat = cat_00.predict(X_test)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "pred_label = preds_cat\n",
    "bag_comb_pred.append(preds_cat)\n",
    "\n",
    "#MLP\n",
    "\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "y_train_boot = boot_df[3].pop('label')\n",
    "X_train_boot = boot_df[3]\n",
    "if 1 == 1 and 0 == 0:\n",
    "    MLP = mlp.fit(X_train_boot, y_train_boot)\n",
    "    y_pred = MLP.predict_proba(X_test)\n",
    "    preds_mlp_00 = np.argmax(y_pred,axis = 1)\n",
    "\n",
    "bag_comb_pred.append(preds_mlp_00)\n",
    "\n",
    "#LGBM\n",
    "\n",
    "lgbm = LGBMClassifier()\n",
    "y_train_boot = boot_df[4].pop('label')\n",
    "X_train_boot = boot_df[4]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    lgbm.fit(X_train_boot, y_train_boot)\n",
    "    preds_lgbm_00 = lgbm.predict(X_test)\n",
    "    bag_comb_pred.append(preds_lgbm_00)\n",
    "#KNN\n",
    "\n",
    "knn_clf_00=KNeighborsClassifier(n_neighbors = 5)\n",
    "y_train_boot = boot_df[5].pop('label')\n",
    "X_train_boot = boot_df[5]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    knn_clf_00.fit(X_train_boot,y_train_boot)\n",
    "if use_model_knn == 1:\n",
    "    preds_knn =knn_clf_00.predict(X_test)\n",
    "    bag_comb_pred.append(preds_knn)\n",
    "#Random Forest\n",
    "\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "y_train_boot = boot_df[6].pop('label')\n",
    "X_train_boot = boot_df[6]\n",
    "\n",
    "if True == True:\n",
    "    model_rf_00 = rf.fit(X_train_boot,y_train_boot)\n",
    "    preds_rf_00 = model_rf_00.predict(X_test)\n",
    "    bag_comb_pred.append(preds_rf_00)\n",
    "#DNN\n",
    "\n",
    "#Model Parameters\n",
    "y_train_boot = boot_df[7].pop('label')\n",
    "X_train_boot = boot_df[7]\n",
    "\n",
    "\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "num_columns = X_train_boot.shape[1]\n",
    "dnn_00 = tf.keras.Sequential()\n",
    "# Input layer\n",
    "dnn_00.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "# Output layer\n",
    "# dnn_00.add(tf.keras.layers.Dense(out_layer))\n",
    "dnn_00.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "dnn_00.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "# Define EarlyStopping callback\n",
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "dnn_00.fit(X_train_boot, y_train_boot, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "pred_dnn = dnn_00.predict(X_test)\n",
    "preds_dnn_00 = np.argmax(pred_dnn,axis = 1)\n",
    "bag_comb_pred.append(preds_dnn_00)\n",
    "#LogReg\n",
    "\n",
    "logreg_00 = LogisticRegression()\n",
    "y_train_boot = boot_df[8].pop('label')\n",
    "X_train_boot = boot_df[8]\n",
    "\n",
    "logreg_00.fit(X_train_boot,y_train_boot)\n",
    "preds_logreg =logreg_00.predict(X_test)\n",
    "bag_comb_pred.append(preds_logreg)\n",
    "\n",
    "y_train_boot = boot_df[9].pop('label')\n",
    "X_train_boot = boot_df[9]\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_boot, label=y_train_boot)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_00 = xgb.train(params, dtrain, num_round)\n",
    "preds_xgb_00 = xgb_00.predict(dtest)\n",
    "bag_comb_pred.append(preds_xgb_00)\n",
    "### 3. The models run in parallel and are independent of each other.\n",
    "bag_vot_df = pd.DataFrame()\n",
    "for i in range(0,len(bag_comb_pred)):\n",
    "    bag_vot_df[f'model_{i}'] =  bag_comb_pred[i]\n",
    "print(bag_vot_df)\n",
    "# Voting start\n",
    "\n",
    "\n",
    "# bag_comb_pred_df = pd.DataFrame(bag_comb_pred)\n",
    "# Extract predictions columns\n",
    "\n",
    "# predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "    # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "predictions = bag_vot_df \n",
    "\n",
    "# predictions = bag_comb_pred_df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "# Use the mode function along axis 1 to get the most common prediction for each row\n",
    "ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "# Add the ensemble predictions to the DataFrame\n",
    "bag_vot_df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "# Display the DataFrame with ensemble predictions\n",
    "print(bag_vot_df)\n",
    "\n",
    "pred_label = bag_vot_df ['ensemble'].values\n",
    "bag_vot_df.pop('ensemble')\n",
    "\n",
    "\n",
    "name='bag_comb'\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "metrics = confusion_metrics(name, pred_label, y_test,time_taken)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating new dataset for level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(len(preds_dnn_prob), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       125646\n",
       "1       139912\n",
       "2        12909\n",
       "3        93898\n",
       "4       135355\n",
       "         ...  \n",
       "4495     97407\n",
       "4496      8154\n",
       "4497     88737\n",
       "4498      5823\n",
       "4499    141488\n",
       "Name: index, Length: 4500, dtype: int64"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_from_series = y_test.to_frame()\n",
    "y_test_reset_index = df_from_series.reset_index()\n",
    "# y_test2 = y_test.reset_index(inplace=True)\n",
    "# print(y_test_reset_index)\n",
    "y_test_reset_index.pop('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_test_reset_index.values[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_dnn_2 = []\n",
    "preds_svm_2 = []\n",
    "preds_rf_2 = []\n",
    "preds_mlp_2 = []\n",
    "preds_ada_2 = []\n",
    "preds_knn_2 = []\n",
    "preds_lgbm_2 = []\n",
    "preds_cat_2 = []\n",
    "preds_xgb_2 = []\n",
    "\n",
    "preds_lr_2 = []\n",
    "preds_dt_2 = []\n",
    "\n",
    "for i in range(0,len(preds_dnn_prob)):  \n",
    "    # print(i)\n",
    "    # print(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_dnn_2.append(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_svm_2.append(preds_svm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_rf_2.append(preds_rf_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_mlp_2.append(preds_mlp_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_ada_2.append(preds_ada_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_knn_2.append(preds_knn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_lgbm_2.append(preds_lgbm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_cat_2.append(preds_cat_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_xgb_2.append(preds_xgb_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_lr_2.append(preds_lr_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_dt_2.append(preds_dt_prob[i][y_test_reset_index.values[i][0]])\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 1. 1. ... 1. 1. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " ...\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n",
      "[[0.8599416  0.94578776 0.99999968 ... 0.99488053 1.         1.        ]\n",
      " [0.88022864 0.93744969 0.99999313 ... 0.94043086 1.         0.        ]\n",
      " [0.92410403 0.76698096 0.99999798 ... 0.99945116 1.         1.        ]\n",
      " ...\n",
      " [0.94917583 0.92804966 0.99999972 ... 0.99861004 1.         1.        ]\n",
      " [0.93674856 0.98348587 0.99999964 ... 0.9912892  1.         0.        ]\n",
      " [0.93769014 0.84863876 0.99999949 ... 0.99555077 1.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('------------START of STRONGER LEARNER - STACK 01 -----------------', file = f)\n",
    "\n",
    "\n",
    "# Stack the vectors horizontally to create a matrix\n",
    "column_features = ['dnn','rf','lgbm','ada','knn','mlp','svm','cat','xgb','lr','dt','label']\n",
    "training_matrix2 = np.column_stack((\n",
    "                          preds_dnn_2,\n",
    "                          preds_rf_2,\n",
    "                          preds_lgbm_2,\n",
    "                          preds_ada_2,\n",
    "                          preds_knn_2, \n",
    "                          preds_mlp_2,\n",
    "                          preds_svm_2,\n",
    "                          preds_cat_2,\n",
    "                          preds_xgb_2,\n",
    "                          preds_lr_2,\n",
    "                          preds_dt_2,\n",
    "                          y_test\n",
    "                          ))\n",
    "\n",
    "training_matrix = np.column_stack((\n",
    "                          preds_dnn,\n",
    "                          preds_rf,\n",
    "                          preds_lgbm,\n",
    "                          preds_ada,\n",
    "                          preds_knn, \n",
    "                          preds_mlp,\n",
    "                          preds_svm,\n",
    "                          preds_cat,\n",
    "                          preds_xgb,\n",
    "                          preds_lr,\n",
    "                          preds_dt,\n",
    "                        #   preds\n",
    "                          y_test\n",
    "                          ))\n",
    "# Print the resulting matrix\n",
    "print(training_matrix)\n",
    "print(training_matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_level_00_0 = pd.DataFrame(training_matrix, columns=column_features)\n",
    "df_level_00_1 = pd.DataFrame(training_matrix2, columns=column_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving the created datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming df is your DataFrame\n",
    "if feature_selection_bit == 1:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_feature_selection.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_feature_selection.csv', index=False)\n",
    "    \n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_all_features.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_all_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.859942</td>\n",
       "      <td>0.945788</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.035412</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.999929e-01</td>\n",
       "      <td>6.606513e-01</td>\n",
       "      <td>0.982959</td>\n",
       "      <td>0.991739</td>\n",
       "      <td>0.994881</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.880229</td>\n",
       "      <td>0.937450</td>\n",
       "      <td>0.999993</td>\n",
       "      <td>0.302884</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.997380e-01</td>\n",
       "      <td>1.333307e-01</td>\n",
       "      <td>0.988308</td>\n",
       "      <td>0.990285</td>\n",
       "      <td>0.940431</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.924104</td>\n",
       "      <td>0.766981</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.085953</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.999975e-01</td>\n",
       "      <td>1.154039e-01</td>\n",
       "      <td>0.973105</td>\n",
       "      <td>0.990019</td>\n",
       "      <td>0.999451</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.945286</td>\n",
       "      <td>0.944461</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.315262</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.999999e-01</td>\n",
       "      <td>8.953582e-02</td>\n",
       "      <td>0.991605</td>\n",
       "      <td>0.977116</td>\n",
       "      <td>0.995532</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.079151</td>\n",
       "      <td>0.767128</td>\n",
       "      <td>0.999956</td>\n",
       "      <td>0.782876</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.856260e-07</td>\n",
       "      <td>3.041613e-11</td>\n",
       "      <td>0.964559</td>\n",
       "      <td>0.982889</td>\n",
       "      <td>0.019402</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4495</th>\n",
       "      <td>0.913306</td>\n",
       "      <td>0.983486</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.380782</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.995879e-01</td>\n",
       "      <td>7.940908e-01</td>\n",
       "      <td>0.995199</td>\n",
       "      <td>0.994188</td>\n",
       "      <td>0.984660</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4496</th>\n",
       "      <td>0.989246</td>\n",
       "      <td>0.993062</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.076962</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>8.867643e-01</td>\n",
       "      <td>0.996495</td>\n",
       "      <td>0.997377</td>\n",
       "      <td>0.999990</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4497</th>\n",
       "      <td>0.949176</td>\n",
       "      <td>0.928050</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.031356</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.386037e-03</td>\n",
       "      <td>0.982680</td>\n",
       "      <td>0.991739</td>\n",
       "      <td>0.998610</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4498</th>\n",
       "      <td>0.936749</td>\n",
       "      <td>0.983486</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.380437</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.997988e-01</td>\n",
       "      <td>9.973323e-01</td>\n",
       "      <td>0.996043</td>\n",
       "      <td>0.993793</td>\n",
       "      <td>0.991289</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4499</th>\n",
       "      <td>0.937690</td>\n",
       "      <td>0.848639</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.310470</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.999830e-01</td>\n",
       "      <td>3.446927e-01</td>\n",
       "      <td>0.984888</td>\n",
       "      <td>0.984617</td>\n",
       "      <td>0.995551</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4500 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           dnn        rf      lgbm       ada  knn           mlp           svm  \\\n",
       "0     0.859942  0.945788  1.000000  0.035412  1.0  9.999929e-01  6.606513e-01   \n",
       "1     0.880229  0.937450  0.999993  0.302884  1.0  9.997380e-01  1.333307e-01   \n",
       "2     0.924104  0.766981  0.999998  0.085953  1.0  9.999975e-01  1.154039e-01   \n",
       "3     0.945286  0.944461  1.000000  0.315262  1.0  9.999999e-01  8.953582e-02   \n",
       "4     0.079151  0.767128  0.999956  0.782876  0.0  7.856260e-07  3.041613e-11   \n",
       "...        ...       ...       ...       ...  ...           ...           ...   \n",
       "4495  0.913306  0.983486  1.000000  0.380782  1.0  9.995879e-01  7.940908e-01   \n",
       "4496  0.989246  0.993062  1.000000  0.076962  1.0  1.000000e+00  8.867643e-01   \n",
       "4497  0.949176  0.928050  1.000000  0.031356  1.0  1.000000e+00  1.386037e-03   \n",
       "4498  0.936749  0.983486  1.000000  0.380437  1.0  9.997988e-01  9.973323e-01   \n",
       "4499  0.937690  0.848639  0.999999  0.310470  1.0  9.999830e-01  3.446927e-01   \n",
       "\n",
       "           cat       xgb        lr   dt  label  \n",
       "0     0.982959  0.991739  0.994881  1.0    1.0  \n",
       "1     0.988308  0.990285  0.940431  1.0    0.0  \n",
       "2     0.973105  0.990019  0.999451  1.0    1.0  \n",
       "3     0.991605  0.977116  0.995532  1.0    0.0  \n",
       "4     0.964559  0.982889  0.019402  1.0    2.0  \n",
       "...        ...       ...       ...  ...    ...  \n",
       "4495  0.995199  0.994188  0.984660  1.0    0.0  \n",
       "4496  0.996495  0.997377  0.999990  1.0    1.0  \n",
       "4497  0.982680  0.991739  0.998610  1.0    1.0  \n",
       "4498  0.996043  0.993793  0.991289  1.0    0.0  \n",
       "4499  0.984888  0.984617  0.995551  1.0    0.0  \n",
       "\n",
       "[4500 rows x 12 columns]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4495</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4496</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4497</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4498</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4499</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4500 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      dnn   rf  lgbm  ada  knn  mlp  svm  cat  xgb   lr   dt  label\n",
       "0     1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "1     0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "2     1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "3     0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "4     1.0  2.0   2.0  2.0  1.0  1.0  1.0  2.0  2.0  1.0  2.0    2.0\n",
       "...   ...  ...   ...  ...  ...  ...  ...  ...  ...  ...  ...    ...\n",
       "4495  0.0  0.0   0.0  1.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "4496  1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "4497  1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "4498  0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "4499  0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "\n",
       "[4500 rows x 12 columns]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now we create performance tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Models   | ACC-00             | PRE-00              | REC-00              | F1-00               |\n",
      "+==========+====================+=====================+=====================+=====================+\n",
      "| ADA      | 0.5397777777777778 | 0.24973841173618946 | 0.30662030712139987 | 0.2329157135863175  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| SVM      | 0.9588888888888889 | 0.9262028343084049  | 0.9101866779825443  | 0.9179051407788913  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DNN      | 0.8604444444444445 | 0.3475140583447532  | 0.38922267895984003 | 0.3669425051275786  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| MLP      | 0.9862222222222222 | 0.8662335767649678  | 0.9710226924322607  | 0.9017144796373003  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| KNN      | 0.9817777777777777 | 0.7705889217922     | 0.7626209303206533  | 0.7665017702069287  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| CAT      | 0.9886666666666667 | 0.7800820189850176  | 0.7700272551134686  | 0.7749584037889947  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| XGB      | 0.9904444444444445 | 0.9784817940829201  | 0.9807944921532954  | 0.9795197389737463  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LGBM     | 0.9942222222222222 | 0.8956940470889165  | 0.9834656564678221  | 0.9227751607758353  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| RF       | 0.9522222222222222 | 0.7745487719830721  | 0.5850000481965352  | 0.597214670613797   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LR       | 0.9724444444444444 | 0.7523498045895212  | 0.7409367323807244  | 0.7464967662382402  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DT       | 0.9857777777777778 | 0.8785755775571594  | 0.973654847931759   | 0.973654847931759   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_svm  | 0.9573333333333334 | 0.9291219466148519  | 0.8941397298834047  | 0.9098655073869489  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_knn  | 0.9804444444444445 | 0.7665521894542537  | 0.7606127092749453  | 0.7634911269342327  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_DT   | 0.9915555555555555 | 0.990498481065547   | 0.9788189444283896  | 0.9845434330490013  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_LR   | 0.9715555555555555 | 0.750875051697339   | 0.7416113444710273  | 0.7461350535549103  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_mlp  | 0.9868888888888889 | 0.8692352689865317  | 0.9736339303673001  | 0.9045771058188462  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_rf   | 0.9468888888888889 | 0.7784428943605695  | 0.5809319994492763  | 0.5985705489787949  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_ada  | 0.6995555555555556 | 0.5626372503314618  | 0.4635364095577231  | 0.45417878409795076 |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_lgbm | 0.98               | 0.7876633010809426  | 0.7241480372450987  | 0.7510435704170609  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_cat  | 0.9868888888888889 | 0.7767212566483375  | 0.7700155131443468  | 0.7732776769788952  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_comb | 0.9755555555555555 | 0.7835227059115569  | 0.7154604253814555  | 0.7442096264332367  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n"
     ]
    }
   ],
   "source": [
    "data = [[\"\" for _ in range(5)] for _ in range(24)]\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'DT',\n",
    "                # 'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "\n",
    "                # 'avg',\n",
    "                # 'weighed_avg'\n",
    "                ]\n",
    "\n",
    "\n",
    "level_00_acc = [ada_acc_00,\n",
    "                svm_acc_00,\n",
    "                dnn_acc_00,\n",
    "                mlp_acc_00,\n",
    "                knn_acc_00,\n",
    "                cat_acc_00,\n",
    "                xgb_acc_00,\n",
    "                lgbm_acc_00,\n",
    "                rf_acc_00,\n",
    "                lr_acc_00,\n",
    "                dt_acc_00,\n",
    "                # voting_acc_00,\n",
    "                bag_svm_acc_00,\n",
    "                bag_knn_acc_00,\n",
    "                bag_dt_acc_00,\n",
    "                bag_lr_acc_00,\n",
    "                bag_mlp_acc_00,\n",
    "               \n",
    "                bag_rf_acc_00,\n",
    "                bag_ada_acc_00,\n",
    "                bag_lgbm_acc_00,\n",
    "\n",
    "                bag_cat_acc_00,\n",
    "                bag_comb_acc_00,\n",
    "               \n",
    "                # avg_acc_00,\n",
    "                # weighed_avg_acc_00\n",
    "                ]  \n",
    "\n",
    "                # ]  \n",
    "\n",
    "level_00_pre = [ada_pre_00,\n",
    "                svm_pre_00,\n",
    "                dnn_pre_00,\n",
    "                mlp_pre_00,\n",
    "                knn_pre_00,\n",
    "                cat_pre_00,\n",
    "                xgb_pre_00,\n",
    "                lgbm_pre_00,\n",
    "                rf_pre_00,\n",
    "                lr_pre_00,\n",
    "                dt_pre_00,\n",
    "                # voting_pre_00,\n",
    "                bag_svm_pre_00,\n",
    "                bag_knn_pre_00,\n",
    "                bag_dt_pre_00,\n",
    "                bag_lr_pre_00,\n",
    "                bag_mlp_pre_00,\n",
    "\n",
    "                bag_rf_pre_00,\n",
    "                bag_ada_pre_00,\n",
    "                bag_lgbm_pre_00,\n",
    "\n",
    "                bag_cat_pre_00,\n",
    "                bag_comb_pre_00,\n",
    "               \n",
    "                # avg_pre_00,\n",
    "                # weighed_avg_pre_00\n",
    "                ]  \n",
    "\n",
    "level_00_rec = [ada_rec_00,\n",
    "                svm_rec_00,\n",
    "                dnn_rec_00,\n",
    "                mlp_rec_00,\n",
    "                knn_rec_00,\n",
    "                cat_rec_00,\n",
    "                xgb_rec_00,\n",
    "                lgbm_rec_00,\n",
    "                rf_rec_00,\n",
    "                lr_rec_00,\n",
    "                dt_rec_00,\n",
    "                # voting_rec_00,\n",
    "                bag_svm_rec_00,\n",
    "                bag_knn_rec_00,\n",
    "                bag_dt_rec_00,\n",
    "                bag_lr_rec_00,\n",
    "                bag_mlp_rec_00,\n",
    "\n",
    "                bag_rf_rec_00,\n",
    "                bag_ada_rec_00,\n",
    "                bag_lgbm_rec_00,\n",
    "\n",
    "                bag_cat_rec_00,\n",
    "                bag_comb_rec_00,\n",
    "               \n",
    "                # avg_rec_00,\n",
    "                # weighed_avg_rec_00\n",
    "                ]  \n",
    "\n",
    "level_00_f1 = [ada_f1_00,\n",
    "                svm_f1_00,\n",
    "                dnn_f1_00,\n",
    "                mlp_f1_00,\n",
    "                knn_f1_00,\n",
    "                cat_f1_00,\n",
    "                xgb_f1_00,\n",
    "                lgbm_f1_00,\n",
    "                rf_f1_00,\n",
    "                lr_f1_00,\n",
    "                dt_rec_00,\n",
    "                # voting_f1_00,\n",
    "                bag_svm_f1_00,\n",
    "                bag_knn_f1_00,\n",
    "                bag_dt_f1_00,\n",
    "                bag_lr_f1_00,\n",
    "                bag_mlp_f1_00,\n",
    "\n",
    "                bag_rf_f1_00,\n",
    "                bag_ada_f1_00,\n",
    "                bag_lgbm_f1_00,\n",
    "\n",
    "                bag_cat_f1_00,\n",
    "                bag_comb_f1_00,\n",
    "               \n",
    "                # avg_f1_00,\n",
    "                # weighed_avg_f1_00\n",
    "                ]                   \n",
    "\n",
    "for i in range(0,len(names_models)):\n",
    "    data[i][0] =  names_models[i]\n",
    "\n",
    "    data[i][1] = level_00_acc[i]\n",
    "    # data[i][2] = level_01_acc[i]\n",
    "\n",
    "    data[i][2] = level_00_pre[i] \n",
    "    # data[i][4] = level_01_pre[i]\n",
    "\n",
    "    data[i][3] = level_00_rec[i] \n",
    "    # data[i][6] = level_01_rec[i]\n",
    "\n",
    "    data[i][4] = level_00_f1[i]\n",
    "    # data[i][8] = level_01_f1[i]\n",
    "\n",
    "\n",
    "headers = [\"Models\", \"ACC-00\",\"PRE-00\",\"REC-00\",\"F1-00\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "print(table)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Models   | ACC-00             | PRE-00              | REC-00              | F1-00               |\n",
      "+==========+====================+=====================+=====================+=====================+\n",
      "| Bag_DT   | 0.9915555555555555 | 0.990498481065547   | 0.9788189444283896  | 0.9845434330490013  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| XGB      | 0.9904444444444445 | 0.9784817940829201  | 0.9807944921532954  | 0.9795197389737463  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DT       | 0.9857777777777778 | 0.8785755775571594  | 0.973654847931759   | 0.973654847931759   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LGBM     | 0.9942222222222222 | 0.8956940470889165  | 0.9834656564678221  | 0.9227751607758353  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| SVM      | 0.9588888888888889 | 0.9262028343084049  | 0.9101866779825443  | 0.9179051407788913  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_svm  | 0.9573333333333334 | 0.9291219466148519  | 0.8941397298834047  | 0.9098655073869489  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_mlp  | 0.9868888888888889 | 0.8692352689865317  | 0.9736339303673001  | 0.9045771058188462  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| MLP      | 0.9862222222222222 | 0.8662335767649678  | 0.9710226924322607  | 0.9017144796373003  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| CAT      | 0.9886666666666667 | 0.7800820189850176  | 0.7700272551134686  | 0.7749584037889947  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_cat  | 0.9868888888888889 | 0.7767212566483375  | 0.7700155131443468  | 0.7732776769788952  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| KNN      | 0.9817777777777777 | 0.7705889217922     | 0.7626209303206533  | 0.7665017702069287  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_knn  | 0.9804444444444445 | 0.7665521894542537  | 0.7606127092749453  | 0.7634911269342327  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_lgbm | 0.98               | 0.7876633010809426  | 0.7241480372450987  | 0.7510435704170609  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LR       | 0.9724444444444444 | 0.7523498045895212  | 0.7409367323807244  | 0.7464967662382402  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_LR   | 0.9715555555555555 | 0.750875051697339   | 0.7416113444710273  | 0.7461350535549103  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_comb | 0.9755555555555555 | 0.7835227059115569  | 0.7154604253814555  | 0.7442096264332367  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_rf   | 0.9468888888888889 | 0.7784428943605695  | 0.5809319994492763  | 0.5985705489787949  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| RF       | 0.9522222222222222 | 0.7745487719830721  | 0.5850000481965352  | 0.597214670613797   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_ada  | 0.6995555555555556 | 0.5626372503314618  | 0.4635364095577231  | 0.45417878409795076 |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DNN      | 0.8604444444444445 | 0.3475140583447532  | 0.38922267895984003 | 0.3669425051275786  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| ADA      | 0.5397777777777778 | 0.24973841173618946 | 0.30662030712139987 | 0.2329157135863175  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n"
     ]
    }
   ],
   "source": [
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_00_acc, level_00_pre, level_00_rec, level_00_f1))\n",
    "\n",
    "# Sort by F1-00 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[4], reverse=True)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_00_acc, sorted_level_00_pre, sorted_level_00_rec, sorted_level_00_f1 = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_00_acc[i]\n",
    "    data[i][2] = sorted_level_00_pre[i] \n",
    "    data[i][3] = sorted_level_00_rec[i] \n",
    "    data[i][4] = sorted_level_00_f1[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"ACC-00\", \"PRE-00\", \"REC-00\", \"F1-00\"]\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Summary table - LEVEL 00', file = f)\n",
    "\n",
    "if feature_selection_bit == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Feature Selection was applied', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('All features were used', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------------+\n",
      "| Models   |   time-00(sec) |\n",
      "+==========+================+\n",
      "| DT       |       0.108712 |\n",
      "+----------+----------------+\n",
      "| RF       |       0.333082 |\n",
      "+----------+----------------+\n",
      "| LR       |       0.454727 |\n",
      "+----------+----------------+\n",
      "| Bag_DT   |       0.599282 |\n",
      "+----------+----------------+\n",
      "| CAT      |       0.704267 |\n",
      "+----------+----------------+\n",
      "| ADA      |       0.746673 |\n",
      "+----------+----------------+\n",
      "| LGBM     |       0.917025 |\n",
      "+----------+----------------+\n",
      "| XGB      |       1.15176  |\n",
      "+----------+----------------+\n",
      "| SVM      |       3.28653  |\n",
      "+----------+----------------+\n",
      "| Bag_rf   |       3.66511  |\n",
      "+----------+----------------+\n",
      "| Bag_LR   |       4.49088  |\n",
      "+----------+----------------+\n",
      "| DNN      |       4.53895  |\n",
      "+----------+----------------+\n",
      "| Bag_ada  |       5.36162  |\n",
      "+----------+----------------+\n",
      "| Bag_svm  |       5.41128  |\n",
      "+----------+----------------+\n",
      "| MLP      |       6.11534  |\n",
      "+----------+----------------+\n",
      "| Bag_cat  |       6.45014  |\n",
      "+----------+----------------+\n",
      "| Bag_lgbm |       7.23719  |\n",
      "+----------+----------------+\n",
      "| KNN      |      10.3394   |\n",
      "+----------+----------------+\n",
      "| Bag_comb |      22.3673   |\n",
      "+----------+----------------+\n",
      "| Bag_knn  |      50.9868   |\n",
      "+----------+----------------+\n",
      "| Bag_mlp  |      92.7483   |\n",
      "+----------+----------------+\n"
     ]
    }
   ],
   "source": [
    "# implement time table\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'DT',\n",
    "                # 'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                # 'avg',\n",
    "                # 'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(2)] for _ in range(len(names_models))]\n",
    "\n",
    "level_00_time = [\n",
    "                ada_time_00,\n",
    "                svm_time_00,\n",
    "                dnn_time_00,\n",
    "                mlp_time_00,\n",
    "                knn_time_00,\n",
    "                cat_time_00,\n",
    "                xgb_time_00,\n",
    "                lgbm_time_00,\n",
    "                rf_time_00,\n",
    "                lr_time_00,\n",
    "                dt_time_00,\n",
    "                # voting_time_00,\n",
    "                bag_svm_time_00,\n",
    "                bag_knn_time_00,\n",
    "                bag_dt_time_00,\n",
    "                bag_lr_time_00,\n",
    "                bag_mlp_time_00,\n",
    "\n",
    "                bag_rf_time_00,\n",
    "                bag_ada_time_00,\n",
    "                bag_lgbm_time_00,\n",
    "                # bag_xgb_time_00,\n",
    "                bag_cat_time_00,\n",
    "                bag_comb_time_00,\n",
    "\n",
    "                # avg_time_00,\n",
    "                # weighed_avg_time_00\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_00_time))\n",
    "\n",
    "# Sort by F1-00 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[1], reverse=False)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_00_time = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_00_time[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"time-00(sec)\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Time is counted is seconds', file = f)\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n",
    "end_program = time.time()\n",
    "time_program = end_program - start_program\n",
    "with open(output_file_name, \"a\") as f: print('Running time of entire program is:', time_program ,' seconds',file = f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
